You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sentry.apache.org by co...@apache.org on 2017/11/20 14:47:28 UTC
[7/9] sentry git commit: SENTRY-2048: Bump Hive version to 2.3.2
(Sergio Pena, reviewed by kalyan kumar kalvagadda, Na Li)
SENTRY-2048: Bump Hive version to 2.3.2 (Sergio Pena, reviewed by kalyan kumar kalvagadda, Na Li)
Project: http://git-wip-us.apache.org/repos/asf/sentry/repo
Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/5f64fe9f
Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/5f64fe9f
Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/5f64fe9f
Branch: refs/heads/akolb-cli
Commit: 5f64fe9f34fb857ccca5a5b09b9ff46300c98674
Parents: a355979
Author: Sergio Pena <se...@cloudera.com>
Authored: Sun Nov 19 10:47:09 2017 -0600
Committer: Sergio Pena <se...@cloudera.com>
Committed: Sun Nov 19 10:47:09 2017 -0600
----------------------------------------------------------------------
pom.xml | 2 +-
.../json/SentryJSONMessageDeserializer.java | 79 +++++++++++++++++++-
.../json/SentryJSONMessageFactory.java | 48 ++++++++++++
.../hive/ql/exec/SentryFilterDDLTask.java | 9 ++-
.../hive/ql/exec/SentryGrantRevokeTask.java | 7 +-
.../binding/hive/HiveAuthzBindingHook.java | 2 +-
.../hive/authz/DefaultSentryValidator.java | 17 +++++
.../hive/authz/HiveAuthzBindingHookBase.java | 10 ---
.../hive/authz/SentryHiveAuthorizerImpl.java | 13 ++++
.../metastore/SentryHiveMetaStoreClient.java | 2 +-
.../TestSentryHiveAuthorizationTaskFactory.java | 16 ++--
.../e2e/hdfs/TestHDFSIntegrationAdvanced.java | 9 ++-
.../tests/e2e/hdfs/TestHDFSIntegrationBase.java | 4 +
.../e2e/hive/TestPrivilegesAtColumnScope.java | 31 ++++----
.../e2e/hive/hiveserver/HiveServerFactory.java | 10 +++
...actMetastoreTestWithStaticConfiguration.java | 3 +-
.../e2e/metastore/TestMetastoreEndToEnd.java | 6 +-
sentry-tests/sentry-tests-solr/pom.xml | 8 ++
18 files changed, 230 insertions(+), 46 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sentry/blob/5f64fe9f/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 6487a26..d863627 100644
--- a/pom.xml
+++ b/pom.xml
@@ -72,7 +72,7 @@ limitations under the License.
<guava.version>14.0.1</guava.version>
<hadoop.version>2.7.2</hadoop.version>
<hamcrest.version>1.3</hamcrest.version>
- <hive.version>2.0.0</hive.version>
+ <hive.version>2.3.2</hive.version>
<jackson.version>1.8.8</jackson.version>
<jdo-api.version>3.0.1</jdo-api.version>
<jetty.version>9.3.21.v20170918</jetty.version>
http://git-wip-us.apache.org/repos/asf/sentry/blob/5f64fe9f/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageDeserializer.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageDeserializer.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageDeserializer.java
index d11b261..929ac8c 100644
--- a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageDeserializer.java
+++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageDeserializer.java
@@ -19,6 +19,11 @@
package org.apache.sentry.binding.metastore.messaging.json;
import org.apache.hive.hcatalog.messaging.*;
+import org.apache.hive.hcatalog.messaging.json.JSONAlterIndexMessage;
+import org.apache.hive.hcatalog.messaging.json.JSONCreateFunctionMessage;
+import org.apache.hive.hcatalog.messaging.json.JSONCreateIndexMessage;
+import org.apache.hive.hcatalog.messaging.json.JSONDropFunctionMessage;
+import org.apache.hive.hcatalog.messaging.json.JSONDropIndexMessage;
import org.apache.hive.hcatalog.messaging.json.JSONInsertMessage;
import org.codehaus.jackson.map.DeserializationConfig;
import org.codehaus.jackson.map.ObjectMapper;
@@ -130,13 +135,81 @@ public class SentryJSONMessageDeserializer extends MessageDeserializer {
}
/**
+ * Method to de-serialize CreateFunctionMessage instance.
+ */
+ @Override
+ public CreateFunctionMessage getCreateFunctionMessage(String messageBody) {
+ // Sentry does not need this message, but it needs to be implemented so that Hive can
+ // complete the notification log for such event.
+ try {
+ return mapper.readValue(messageBody, JSONCreateFunctionMessage.class);
+ } catch (Exception e) {
+ throw new IllegalArgumentException("Could not construct JSONCreateFunctionMessage: ", e);
+ }
+ }
+
+ /**
+ * Method to de-serialize DropFunctionMessage instance.
+ */
+ @Override
+ public DropFunctionMessage getDropFunctionMessage(String messageBody) {
+ // Sentry does not need this message, but it needs to be implemented so that Hive can
+ // complete the notification log for such event.
+ try {
+ return mapper.readValue(messageBody, JSONDropFunctionMessage.class);
+ } catch (Exception e) {
+ throw new IllegalArgumentException("Could not construct JSONDropDatabaseMessage: ", e);
+ }
+ }
+
+ /**
+ * Method to de-serialize CreateIndexMessage instance. + */
+ @Override
+ public CreateIndexMessage getCreateIndexMessage(String messageBody) {
+ // Sentry does not need this message, but it needs to be implemented so that Hive can
+ // complete the notification log for such event.
+ try {
+ return mapper.readValue(messageBody, JSONCreateIndexMessage.class);
+ } catch (Exception e) {
+ throw new IllegalArgumentException("Could not construct JSONCreateIndexMessage: ", e);
+ }
+ }
+
+ /**
+ * Method to de-serialize DropIndexMessage instance.
+ */
+ @Override
+ public DropIndexMessage getDropIndexMessage(String messageBody) {
+ // Sentry does not need this message, but it needs to be implemented so that Hive can
+ // complete the notification log for such event.
+ try {
+ return mapper.readValue(messageBody, JSONDropIndexMessage.class);
+ } catch (Exception e) {
+ throw new IllegalArgumentException("Could not construct JSONDropIndexMessage: ", e);
+ }
+ }
+
+ /**
+ * Method to de-serialize AlterIndexMessage instance.
+ */
+ @Override
+ public AlterIndexMessage getAlterIndexMessage(String messageBody) {
+ // Sentry does not need this message, but it needs to be implemented so that Hive can
+ // complete the notification log for such event.
+ try {
+ return mapper.readValue(messageBody, JSONAlterIndexMessage.class);
+ } catch (Exception e) {
+ throw new IllegalArgumentException("Could not construct JSONAlterIndexMessage: ", e);
+ }
+ }
+
+ /**
* Method to de-serialize JSONInsertMessage instance.
*/
@Override
public InsertMessage getInsertMessage(String messageBody) {
- // Sentry would be not be interested in InsertMessage as these are generated when is data is
- // added inserted. This method is implemented for completeness. This is reason why, new sentry
- // JSON class is not defined for InsertMessage.
+ // Sentry does not need this message, but it needs to be implemented so that Hive can
+ // complete the notification log for such event.
try {
return mapper.readValue(messageBody, JSONInsertMessage.class);
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/sentry/blob/5f64fe9f/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageFactory.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageFactory.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageFactory.java
index b531976..0af02d1 100644
--- a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageFactory.java
+++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageFactory.java
@@ -22,9 +22,16 @@ import com.google.common.collect.Lists;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.Function;
+import org.apache.hadoop.hive.metastore.api.Index;
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hive.hcatalog.messaging.*;
+import org.apache.hive.hcatalog.messaging.json.JSONAlterIndexMessage;
+import org.apache.hive.hcatalog.messaging.json.JSONCreateFunctionMessage;
+import org.apache.hive.hcatalog.messaging.json.JSONCreateIndexMessage;
+import org.apache.hive.hcatalog.messaging.json.JSONDropFunctionMessage;
+import org.apache.hive.hcatalog.messaging.json.JSONDropIndexMessage;
import org.apache.hive.hcatalog.messaging.json.JSONInsertMessage;
import java.util.*;
@@ -108,6 +115,47 @@ public class SentryJSONMessageFactory extends MessageFactory {
}
@Override
+ public CreateFunctionMessage buildCreateFunctionMessage(Function function) {
+ // Sentry would be not be interested in CreateFunctionMessage as these are generated when is data is
+ // added inserted. This method is implemented for completeness. This is reason why, new sentry
+ // JSON class is not defined for CreateFunctionMessage
+ return new JSONCreateFunctionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, function, now());
+ }
+
+ @Override
+ public DropFunctionMessage buildDropFunctionMessage(Function function) {
+ // Sentry would be not be interested in DropFunctionMessage as these are generated when is data is
+ // added inserted. This method is implemented for completeness. This is reason why, new sentry
+ // JSON class is not defined for DropFunctionMessage
+ return new JSONDropFunctionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, function, now());
+
+ }
+
+ @Override
+ public CreateIndexMessage buildCreateIndexMessage(Index index) {
+ // Sentry would be not be interested in CreateIndexMessage as these are generated when is data is
+ // added inserted. This method is implemented for completeness. This is reason why, new sentry
+ // JSON class is not defined for CreateIndexMessage
+ return new JSONCreateIndexMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, index, now());
+ }
+
+ @Override
+ public DropIndexMessage buildDropIndexMessage(Index index) {
+ // Sentry would be not be interested in DropIndexMessage as these are generated when is data is
+ // added inserted. This method is implemented for completeness. This is reason why, new sentry
+ // JSON class is not defined for DropIndexMessage
+ return new JSONDropIndexMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, index, now());
+ }
+
+ @Override
+ public AlterIndexMessage buildAlterIndexMessage(Index before, Index after) {
+ // Sentry would be not be interested in AlterIndexMessage as these are generated when is data is
+ // added inserted. This method is implemented for completeness. This is reason why, new sentry
+ // JSON class is not defined for AlterIndexMessage
+ return new JSONAlterIndexMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, before, after, now());
+ }
+
+ @Override
public InsertMessage buildInsertMessage(String db, String table, Map<String,String> partKeyVals,
List<String> files) {
// Sentry would be not be interested in InsertMessage as these are generated when is data is
http://git-wip-us.apache.org/repos/asf/sentry/blob/5f64fe9f/sentry-binding/sentry-binding-hive/src/main/java/org/apache/hadoop/hive/ql/exec/SentryFilterDDLTask.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/hadoop/hive/ql/exec/SentryFilterDDLTask.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/hadoop/hive/ql/exec/SentryFilterDDLTask.java
index 672acb6..efe113e 100644
--- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/hadoop/hive/ql/exec/SentryFilterDDLTask.java
+++ b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/hadoop/hive/ql/exec/SentryFilterDDLTask.java
@@ -143,9 +143,6 @@ public class SentryFilterDDLTask extends DDLTask {
parentTasks = ddlTask.getParentTasks();
backupTask = ddlTask.getBackupTask();
backupChildrenTasks = ddlTask.getBackupChildrenTasks();
- started = ddlTask.started();
- isdone = ddlTask.done();
- queued = ddlTask.getQueued();
id = ddlTask.getId();
taskCounters = ddlTask.getCounters();
feedSubscribers = ddlTask.getFeedSubscribers();
@@ -157,5 +154,11 @@ public class SentryFilterDDLTask extends DDLTask {
setException(ddlTask.getException());
console = ddlTask.console;
setFetchSource(ddlTask.isFetchSource());
+ taskHandle = ddlTask.getTaskHandle();
+ conf = ddlTask.conf;
+ queryState = ddlTask.queryState;
+ driverContext = ddlTask.getDriverContext();
+ clonedConf = ddlTask.clonedConf;
+ queryDisplay = ddlTask.queryDisplay;
}
}
http://git-wip-us.apache.org/repos/asf/sentry/blob/5f64fe9f/sentry-binding/sentry-binding-hive/src/main/java/org/apache/hadoop/hive/ql/exec/SentryGrantRevokeTask.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/hadoop/hive/ql/exec/SentryGrantRevokeTask.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/hadoop/hive/ql/exec/SentryGrantRevokeTask.java
index 61052cf..21a6abf 100644
--- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/hadoop/hive/ql/exec/SentryGrantRevokeTask.java
+++ b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/hadoop/hive/ql/exec/SentryGrantRevokeTask.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.hive.metastore.api.PrincipalType;
import org.apache.hadoop.hive.ql.CompilationOpContext;
import org.apache.hadoop.hive.ql.DriverContext;
import org.apache.hadoop.hive.ql.QueryPlan;
+import org.apache.hadoop.hive.ql.QueryState;
import org.apache.hadoop.hive.ql.hooks.ReadEntity;
import org.apache.hadoop.hive.ql.hooks.WriteEntity;
import org.apache.hadoop.hive.ql.metadata.AuthorizationException;
@@ -109,11 +110,11 @@ public class SentryGrantRevokeTask extends Task<DDLWork> implements Serializable
private HiveOperation stmtOperation;
@Override
- public void initialize(HiveConf conf, QueryPlan queryPlan, DriverContext ctx,
+ public void initialize(QueryState queryState, QueryPlan queryPlan, DriverContext ctx,
CompilationOpContext opContext) {
// CompilationOpContext is an unused parameter on the initialize() method.
- super.initialize(conf, queryPlan, driverContext, null);
- this.conf = conf;
+ super.initialize(queryState, queryPlan, driverContext, null);
+ this.conf = queryState.getConf();
}
@Override
http://git-wip-us.apache.org/repos/asf/sentry/blob/5f64fe9f/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHook.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHook.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHook.java
index 802bf9c..e4620ea 100644
--- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHook.java
+++ b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHook.java
@@ -291,7 +291,7 @@ public class HiveAuthzBindingHook extends HiveAuthzBindingHookBase {
@Override
public void postAnalyze(HiveSemanticAnalyzerHookContext context,
List<Task<? extends Serializable>> rootTasks) throws SemanticException {
- HiveOperation stmtOperation = getCurrentHiveStmtOp();
+ HiveOperation stmtOperation = context.getHiveOperation();
HiveAuthzPrivileges stmtAuthObject;
stmtAuthObject = HiveAuthzPrivilegesMap.getHiveAuthzPrivileges(stmtOperation);
http://git-wip-us.apache.org/repos/asf/sentry/blob/5f64fe9f/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/authz/DefaultSentryValidator.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/authz/DefaultSentryValidator.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/authz/DefaultSentryValidator.java
index d1f071e..319a1be 100644
--- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/authz/DefaultSentryValidator.java
+++ b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/authz/DefaultSentryValidator.java
@@ -27,6 +27,7 @@ import java.util.List;
import java.util.Set;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.metadata.AuthorizationException;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.plan.HiveOperation;
import org.apache.hadoop.hive.ql.security.HiveAuthenticationProvider;
import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAccessControlException;
@@ -375,6 +376,22 @@ public class DefaultSentryValidator extends SentryHiveAuthorizationValidator {
return listObjs;
}
+ @Override
+ public List<HivePrivilegeObject> applyRowFilterAndColumnMasking(HiveAuthzContext hiveAuthzContext,
+ List<HivePrivilegeObject> list) throws SemanticException {
+ // Sentry does not support this feature yet. Returning null is enough to let Hive
+ // that no row filtering nor column masking will be applied.
+ return null;
+ }
+
+ @Override
+ public boolean needTransform() {
+ // Hive uses this value to know whether a Hive query must be transformed if row filtering
+ // or column masking is applied. Sentry does not support such feature yet, so returning
+ // false is enough to let Hive know that the query is not required to be transformed.
+ return false;
+ }
+
private List<HivePrivilegeObject> filterShowTables(List<HivePrivilegeObject> listObjs,
String userName, HiveAuthzBinding hiveAuthzBinding) {
List<HivePrivilegeObject> filteredResult = new ArrayList<HivePrivilegeObject>();
http://git-wip-us.apache.org/repos/asf/sentry/blob/5f64fe9f/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/authz/HiveAuthzBindingHookBase.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/authz/HiveAuthzBindingHookBase.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/authz/HiveAuthzBindingHookBase.java
index 2e299a9..9c60c22 100644
--- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/authz/HiveAuthzBindingHookBase.java
+++ b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/authz/HiveAuthzBindingHookBase.java
@@ -886,18 +886,8 @@ public abstract class HiveAuthzBindingHookBase extends AbstractSemanticAnalyzerH
}
}
- protected HiveOperation getCurrentHiveStmtOp() {
- SessionState sessState = SessionState.get();
- if (sessState == null) {
- // TODO: Warn
- return null;
- }
- return sessState.getHiveOperation();
- }
-
protected Subject getCurrentSubject(HiveSemanticAnalyzerHookContext context) {
// Extract the username from the hook context
return new Subject(context.getUserName());
}
-
}
http://git-wip-us.apache.org/repos/asf/sentry/blob/5f64fe9f/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/authz/SentryHiveAuthorizerImpl.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/authz/SentryHiveAuthorizerImpl.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/authz/SentryHiveAuthorizerImpl.java
index 1596bce..86ff0cc 100644
--- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/authz/SentryHiveAuthorizerImpl.java
+++ b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/authz/SentryHiveAuthorizerImpl.java
@@ -20,11 +20,13 @@ import java.util.List;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.SentryHivePrivilegeObjectDesc;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.plan.PrivilegeObjectDesc;
import org.apache.hadoop.hive.ql.security.authorization.DefaultHiveAuthorizationTranslator;
import org.apache.hadoop.hive.ql.security.authorization.plugin.AbstractHiveAuthorizer;
import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAccessControlException;
import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthorizationTranslator;
+import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthorizer.VERSION;
import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthzContext;
import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthzPluginException;
import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveOperationType;
@@ -165,6 +167,17 @@ public class SentryHiveAuthorizerImpl extends AbstractHiveAuthorizer {
return hiveTranslator;
}
+ @Override
+ public List<HivePrivilegeObject> applyRowFilterAndColumnMasking(HiveAuthzContext hiveAuthzContext,
+ List<HivePrivilegeObject> list) throws SemanticException {
+ return null;
+ }
+
+ @Override
+ public boolean needTransform() {
+ return false;
+ }
+
protected static HivePrivilegeObjectType getPrivObjectType(
SentryHivePrivilegeObjectDesc privSubjectDesc) {
if (privSubjectDesc.getObject() == null) {
http://git-wip-us.apache.org/repos/asf/sentry/blob/5f64fe9f/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentryHiveMetaStoreClient.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentryHiveMetaStoreClient.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentryHiveMetaStoreClient.java
index cfc5c04..e30a860 100644
--- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentryHiveMetaStoreClient.java
+++ b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentryHiveMetaStoreClient.java
@@ -49,7 +49,7 @@ public class SentryHiveMetaStoreClient extends HiveMetaStoreClient implements
public SentryHiveMetaStoreClient(HiveConf conf, HiveMetaHookLoader hookLoader)
throws MetaException {
- super(conf, hookLoader);
+ super(conf, hookLoader, true);
}
@Override
http://git-wip-us.apache.org/repos/asf/sentry/blob/5f64fe9f/sentry-binding/sentry-binding-hive/src/test/java/org/apache/sentry/binding/hive/TestSentryHiveAuthorizationTaskFactory.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive/src/test/java/org/apache/sentry/binding/hive/TestSentryHiveAuthorizationTaskFactory.java b/sentry-binding/sentry-binding-hive/src/test/java/org/apache/sentry/binding/hive/TestSentryHiveAuthorizationTaskFactory.java
index de073ed..c75f57d 100644
--- a/sentry-binding/sentry-binding-hive/src/test/java/org/apache/sentry/binding/hive/TestSentryHiveAuthorizationTaskFactory.java
+++ b/sentry-binding/sentry-binding-hive/src/test/java/org/apache/sentry/binding/hive/TestSentryHiveAuthorizationTaskFactory.java
@@ -22,6 +22,7 @@ import java.io.Serializable;
import java.util.HashMap;
import java.util.List;
+import org.apache.hadoop.hive.ql.QueryState;
import org.junit.Assert;
import org.apache.commons.io.FileUtils;
@@ -37,7 +38,6 @@ import org.apache.hadoop.hive.ql.metadata.Partition;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.parse.ASTNode;
import org.apache.hadoop.hive.ql.parse.DDLSemanticAnalyzer;
-import org.apache.hadoop.hive.ql.parse.ParseDriver;
import org.apache.hadoop.hive.ql.parse.ParseUtils;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.plan.DDLWork;
@@ -70,7 +70,6 @@ public class TestSentryHiveAuthorizationTaskFactory {
private static final String SERVER = "server1";
- private ParseDriver parseDriver;
private DDLSemanticAnalyzer analyzer;
private HiveConf conf;
private Context context;
@@ -91,12 +90,19 @@ public class TestSentryHiveAuthorizationTaskFactory {
conf.setVar(ConfVars.HIVE_AUTHORIZATION_TASK_FACTORY,
SentryHiveAuthorizationTaskFactoryImpl.class.getName());
+ // This configuration avoids starting the HS2 WebUI which was causes test failures when
+ // HS2 is configured for concurrency
+ conf.setBoolVar(HiveConf.ConfVars.HIVE_IN_TEST, true);
+
+ // This configuration avoids that the HMS fails if the Metastore schema has not version
+ // information. For some reason, HMS does not set a version initially on our tests.
+ conf.setBoolVar(HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION, false);
+
db = Mockito.mock(Hive.class);
table = new Table(DB, TABLE);
partition = new Partition(table);
context = new Context(conf);
- parseDriver = new ParseDriver();
- analyzer = new DDLSemanticAnalyzer(conf, db);
+ analyzer = new DDLSemanticAnalyzer(new QueryState(conf), db);
SessionState.start(conf);
Mockito.when(db.getTable(TABLE, false)).thenReturn(table);
Mockito.when(db.getPartition(table, new HashMap<String, String>(), false))
@@ -487,7 +493,7 @@ public class TestSentryHiveAuthorizationTaskFactory {
}
private ASTNode parse(String command) throws Exception {
- return ParseUtils.findRootNonNullToken(parseDriver.parse(command));
+ return ParseUtils.parse(command);
}
private DDLWork analyze(ASTNode ast) throws Exception {
http://git-wip-us.apache.org/repos/asf/sentry/blob/5f64fe9f/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationAdvanced.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationAdvanced.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationAdvanced.java
index 95bbaeb..33ace57 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationAdvanced.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationAdvanced.java
@@ -791,7 +791,14 @@ public class TestHDFSIntegrationAdvanced extends TestHDFSIntegrationBase {
// Alter table tab1 to be tbCopy which is at scheme-less location.
// And the corresponding path will be updated to sentry server.
hmsClient.alter_table(dbName, "tab1", tbCopy);
- Assert.assertEquals(hmsClient.getTable(dbName, tblName).getSd().getLocation(), "/tmp/external");
+
+ // Remove the checking for the location of the table. The HMS will never return scheme-less
+ // URI locations anymore. However, if any NPE being triggered in future because of any changes,
+ // the test case will cover it and capture it.
+ // i.e. hdfs://<localhost>/tmp/external (location with scheme)
+ // /tmp/external (location without scheme)
+ // Assert.assertEquals("/tmp/external", hmsClient.getTable(dbName, tblName).getSd().getLocation());
+
verifyOnPath("/tmp/external", FsAction.ALL, StaticUserGroup.HIVE, true);
stmt.close();
http://git-wip-us.apache.org/repos/asf/sentry/blob/5f64fe9f/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationBase.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationBase.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationBase.java
index 7495fc7..25a678b 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationBase.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationBase.java
@@ -575,6 +575,10 @@ public abstract class TestHDFSIntegrationBase {
String hadoopTempDir = System.getProperty("java.io.tmpdir") + File.separator + "hadoop-tmp";
hiveConf.set("hadoop.tmp.dir", hadoopTempDir);
+ // This configuration will avoid that the HMS fails if the metastore schema has not version
+ // information. For some reason, HMS does not set a version initially on our tests.
+ hiveConf.set(ConfVars.METASTORE_SCHEMA_VERIFICATION.varname, "false");
+
// Sets hive.metastore.authorization.storage.checks to true, so that
// disallow the operations such as drop-partition if the user in question
// doesn't have permissions to delete the corresponding directory
http://git-wip-us.apache.org/repos/asf/sentry/blob/5f64fe9f/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestPrivilegesAtColumnScope.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestPrivilegesAtColumnScope.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestPrivilegesAtColumnScope.java
index c2fee2a..9d820a8 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestPrivilegesAtColumnScope.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestPrivilegesAtColumnScope.java
@@ -205,12 +205,13 @@ public class TestPrivilegesAtColumnScope extends AbstractTestWithStaticConfigura
context.verifyAuthzException(e);
}
- // negative test: test user can't query column of view
+ // positive test: test user can query column of view
+ // Hive 2 adds a new column view authorization as part of the Column Pruning feature
+ // See Hive ColumnPrunerSelectProc.process() on how view authorization is handled
try {
statement.execute("SELECT COUNT(A) FROM VIEW_1");
- Assert.fail("Expected SQL exception");
} catch (SQLException e) {
- context.verifyAuthzException(e);
+ Assert.fail("Exception not expected.");
}
// negative test: test user can't query column of view
try {
@@ -255,18 +256,18 @@ public class TestPrivilegesAtColumnScope extends AbstractTestWithStaticConfigura
context.verifyAuthzException(e);
}
- // negative test: test user can't query view
+ // positive test: test user can query column of view
+ // Hive 2 adds a new column view authorization as part of the Column Pruning feature
+ // See Hive ColumnPrunerSelectProc.process() on how view authorization is handled
try {
statement.execute("SELECT COUNT(A) FROM VIEW_1");
- Assert.fail("Expected SQL exception");
} catch (SQLException e) {
- context.verifyAuthzException(e);
+ Assert.fail("Exception not expected.");
}
try {
statement.execute("SELECT COUNT(B) FROM VIEW_1");
- Assert.fail("Expected SQL exception");
} catch (SQLException e) {
- context.verifyAuthzException(e);
+ Assert.fail("Exception not expected.");
}
// negative test: test user can't create a new view
@@ -378,12 +379,13 @@ public class TestPrivilegesAtColumnScope extends AbstractTestWithStaticConfigura
Statement statement = context.createStatement(connection);
statement.execute("USE DB_1");
- // test user can't execute query VIEW_1 JOIN VIEW_2
+ // test user can execute query VIEW_1 JOIN VIEW_2
+ // Hive 2 adds a new column view authorization as part of the Column Pruning feature
+ // See Hive ColumnPrunerSelectProc.process() on how view authorization is handled
try {
statement.execute("SELECT COUNT(*) FROM VIEW_1 V1 JOIN VIEW_2 V2 ON (V1.B = V2.B)");
- Assert.fail("Expected SQL Exception");
} catch (SQLException e) {
- context.verifyAuthzException(e);
+ Assert.fail("Exception not expected");
}
// test user can't execute query VIEW_1 JOIN TAB_2
@@ -438,12 +440,13 @@ public class TestPrivilegesAtColumnScope extends AbstractTestWithStaticConfigura
context.verifyAuthzException(e);
}
- // test user can't execute query VIEW_1 JOIN VIEW_2
+ // test user can execute query VIEW_1 JOIN VIEW_2
+ // Hive 2 adds a new column view authorization as part of the Column Pruning feature
+ // See Hive ColumnPrunerSelectProc.process() on how view authorization is handled
try {
statement.execute("SELECT COUNT(*) FROM VIEW_1 V1 JOIN VIEW_2 V2 ON (V1.B = V2.B)");
- Assert.fail("Expected SQL Exception");
} catch (SQLException e) {
- context.verifyAuthzException(e);
+ Assert.fail("Exception not expected");
}
// test user can't execute query TAB_1 JOIN TAB_2
http://git-wip-us.apache.org/repos/asf/sentry/blob/5f64fe9f/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/hiveserver/HiveServerFactory.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/hiveserver/HiveServerFactory.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/hiveserver/HiveServerFactory.java
index d5eb137..7d41348 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/hiveserver/HiveServerFactory.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/hiveserver/HiveServerFactory.java
@@ -206,6 +206,16 @@ public class HiveServerFactory {
String hadoopTempDir = System.getProperty("java.io.tmpdir") + File.separator + "hadoop-tmp";
properties.put("hadoop.tmp.dir", hadoopTempDir);
+ // This configuration will avoid that the HMS fails if the metastore schema has not version
+ // information. For some reason, HMS does not set a version initially on our tests.
+ properties.put(ConfVars.METASTORE_SCHEMA_VERIFICATION.varname, "false");
+
+ // Disable join cartesian checks to allow Sentry tests to pass
+ properties.put(ConfVars.HIVE_STRICT_CHECKS_CARTESIAN.varname, "false");
+
+ // Disable capability checks (these checks do not work when Hive is in testing mode)
+ properties.put(ConfVars.METASTORE_CAPABILITY_CHECK.varname, "false");
+
if (!properties.containsKey(METASTORE_BYPASS)) {
properties.put(METASTORE_BYPASS, "hive,impala," + System.getProperty("user.name", ""));
} else {
http://git-wip-us.apache.org/repos/asf/sentry/blob/5f64fe9f/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/AbstractMetastoreTestWithStaticConfiguration.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/AbstractMetastoreTestWithStaticConfiguration.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/AbstractMetastoreTestWithStaticConfiguration.java
index d0139ad..f14cbb6 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/AbstractMetastoreTestWithStaticConfiguration.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/AbstractMetastoreTestWithStaticConfiguration.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.hive.cli.CliSessionState;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.EnvironmentContext;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Order;
import org.apache.hadoop.hive.metastore.api.Partition;
@@ -138,7 +139,7 @@ public abstract class AbstractMetastoreTestWithStaticConfiguration extends
public void alterPartitionWithLocation(HiveMetaStoreClient client, Partition partition, String location) throws Exception {
partition.getSd().setLocation(location);
- client.alter_partition(partition.getDbName(), partition.getTableName(), partition);
+ client.alter_partition(partition.getDbName(), partition.getTableName(), partition, new EnvironmentContext());
}
public void renamePartition(HiveMetaStoreClient client, Partition partition, Partition newPartition) throws Exception {
http://git-wip-us.apache.org/repos/asf/sentry/blob/5f64fe9f/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestMetastoreEndToEnd.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestMetastoreEndToEnd.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestMetastoreEndToEnd.java
index 7f53527..f8f304f 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestMetastoreEndToEnd.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestMetastoreEndToEnd.java
@@ -507,14 +507,14 @@ public class TestMetastoreEndToEnd extends
client = context.getMetaStoreClient(USER1_1);
Partition newPartition = client.getPartition(dbName, tabName1, partVals1);
newPartition.getSd().setLocation(tabDir1);
- client.alter_partition(dbName, tabName1, newPartition);
+ client.alter_partition(dbName, tabName1, newPartition, null);
client.close();
// user with Table and URI privileges should be able to alter partition set location
client = context.getMetaStoreClient(USER2_1);
newPartition = client.getPartition(dbName, tabName1, partVals2);
newPartition.getSd().setLocation(tabDir1);
- client.alter_partition(dbName, tabName1, newPartition);
+ client.alter_partition(dbName, tabName1, newPartition, null);
client.close();
policyFile.addRolesToGroup(USERGROUP3, db_all_role);
@@ -524,7 +524,7 @@ public class TestMetastoreEndToEnd extends
newPartition = client.getPartition(dbName, tabName1, partVals2);
newPartition.getSd().setLocation(tabDir1);
try {
- client.alter_partition(dbName, tabName1, newPartition);
+ client.alter_partition(dbName, tabName1, newPartition, null);
fail("alter partition with location should have failed");
} catch (MetaException e) {
Context.verifyMetastoreAuthException(e);
http://git-wip-us.apache.org/repos/asf/sentry/blob/5f64fe9f/sentry-tests/sentry-tests-solr/pom.xml
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-solr/pom.xml b/sentry-tests/sentry-tests-solr/pom.xml
index 723fa9d..9d400b2 100644
--- a/sentry-tests/sentry-tests-solr/pom.xml
+++ b/sentry-tests/sentry-tests-solr/pom.xml
@@ -110,6 +110,10 @@ limitations under the License.
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-shims</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -195,6 +199,10 @@ limitations under the License.
<groupId>org.eclipse.jetty.aggregate</groupId>
<artifactId>jetty-all</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-llap-server</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>