You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sentry.apache.org by ha...@apache.org on 2017/01/27 23:20:54 UTC
[7/7] sentry git commit: SENTRY-1587
SENTRY-1587
Change-Id: Ic21fdff7d7fd54d785683f0ec3dd3598cec33f74
Project: http://git-wip-us.apache.org/repos/asf/sentry/repo
Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/96e1d9a1
Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/96e1d9a1
Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/96e1d9a1
Branch: refs/heads/sentry-ha-redesign-1
Commit: 96e1d9a1cc272c9c044f68034648300fcf9b8477
Parents: 18be1d5
Author: hahao <ha...@cloudera.com>
Authored: Fri Jan 27 15:00:36 2017 -0800
Committer: hahao <ha...@cloudera.com>
Committed: Fri Jan 27 15:00:36 2017 -0800
----------------------------------------------------------------------
.../db/service/persistent/SentryStore.java | 62 +++++++++++++++++---
.../sentry/service/thrift/HMSFollower.java | 56 ++++++++++++++++--
.../db/service/persistent/TestSentryStore.java | 11 ++++
3 files changed, 116 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sentry/blob/96e1d9a1/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java
index d1edcb1..9019350 100644
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java
@@ -2269,13 +2269,12 @@ public class SentryStore {
return (PathsImage) tm.executeTransaction(
new TransactionBlock() {
public Object execute(PersistenceManager pm) throws Exception {
- long curChangeID = getLastProcessedPermChangeIDCore(pm);
+
+ // curChangeID could be 0, since first full snapshot is fetching
+ // from HMS, and does not have corresponding delta update.
+ long curChangeID = getLastProcessedPathChangeIDCore(pm);
Map<String, Set<String>> pathImage = retrieveFullPathsImageCore(pm);
- if (curChangeID == EMPTY_CHANGE_ID && !pathImage.isEmpty()) {
- throw new Exception("Non-empty full paths image should not have" +
- "an empty change ID.");
- }
return new PathsImage(pathImage, curChangeID);
}
});
@@ -2331,7 +2330,7 @@ public class SentryStore {
private void createAuthzPathsMappingCore(PersistenceManager pm, String authzObj,
Set<String> paths) throws SentryAlreadyExistsException {
- MAuthzPathsMapping mAuthzPathsMapping = getMAuthzPathsMapping(pm, authzObj);
+ MAuthzPathsMapping mAuthzPathsMapping = getMAuthzPathsMappingCore(pm, authzObj);
if (mAuthzPathsMapping == null) {
mAuthzPathsMapping =
@@ -2342,10 +2341,59 @@ public class SentryStore {
}
}
+
+ public void updateAuthzPathsMapping(final String hiveObj, final Set<String> paths)
+ throws Exception {
+ tm.executeTransactionWithRetry (
+ new TransactionBlock() {
+ public Object execute(PersistenceManager pm) throws Exception {
+ updateAuthzPathsMappingCore(pm, hiveObj, paths);
+ return null;
+ }
+ });
+ }
+
+ public void updateAuthzPathsMapping(final String hiveObj, final Set<String> paths,
+ final DeltaTransactionBlock deltaTransactionBlock) throws Exception {
+ execute(deltaTransactionBlock, new TransactionBlock() {
+ public Object execute(PersistenceManager pm) throws Exception {
+ updateAuthzPathsMappingCore(pm, hiveObj, paths);
+ return null;
+ }
+ });
+ }
+
+ private void updateAuthzPathsMappingCore(PersistenceManager pm, String authzObj,
+ Set<String> paths) throws SentryAlreadyExistsException {
+
+ MAuthzPathsMapping mAuthzPathsMapping = getMAuthzPathsMappingCore(pm, authzObj);
+
+ if (mAuthzPathsMapping == null) {
+ mAuthzPathsMapping =
+ new MAuthzPathsMapping(authzObj, paths, System.currentTimeMillis());
+ } else {
+ mAuthzPathsMapping.getPaths().addAll(paths);
+ }
+ pm.makePersistent(mAuthzPathsMapping);
+ }
+
+ /**
+ * Get the MAuthzPathsMapping object from authzObj
+ */
+ public MAuthzPathsMapping getMAuthzPathsMapping(final String authzObj)
+ throws Exception{
+ return tm.executeTransactionWithRetry(
+ new TransactionBlock<MAuthzPathsMapping>() {
+ public MAuthzPathsMapping execute(PersistenceManager pm) throws Exception {
+ return getMAuthzPathsMappingCore(pm, authzObj);
+ }
+ });
+ }
+
/**
* Get the MAuthzPathsMapping object from authzObj
*/
- public MAuthzPathsMapping getMAuthzPathsMapping(PersistenceManager pm, String authzObj) {
+ private MAuthzPathsMapping getMAuthzPathsMappingCore(PersistenceManager pm, String authzObj) {
Query query = pm.newQuery(MAuthzPathsMapping.class);
query.setFilter("this.authzObjName == t");
query.declareParameters("java.lang.String t");
http://git-wip-us.apache.org/repos/asf/sentry/blob/96e1d9a1/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java
index ad6bdda..7dce745 100644
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java
@@ -19,6 +19,7 @@ package org.apache.sentry.service.thrift;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
+import com.google.common.collect.Sets;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
@@ -32,8 +33,8 @@ import org.apache.sentry.binding.hive.conf.HiveAuthzConf;
import org.apache.sentry.core.common.exception.*;
import org.apache.sentry.hdfs.PathsUpdate;
import org.apache.sentry.hdfs.PermissionsUpdate;
-import org.apache.sentry.hdfs.UpdateableAuthzPaths;
import org.apache.sentry.hdfs.FullUpdateInitializer;
+import org.apache.sentry.hdfs.SentryMalformedPathException;
import org.apache.sentry.hdfs.service.thrift.TPrivilegeChanges;
import org.apache.sentry.provider.db.SentryPolicyStorePlugin;
import org.apache.sentry.provider.db.service.persistent.DeltaTransactionBlock;
@@ -321,12 +322,12 @@ public class HMSFollower implements Runnable {
/**
* Throws SentryInvalidHMSEventException if Notification event contains insufficient information
*/
- void processNotificationEvents(List<NotificationEvent> events) throws
- SentryInvalidHMSEventException, SentryInvalidInputException {
+ void processNotificationEvents(List<NotificationEvent> events) throws Exception {
SentryJSONMessageDeserializer deserializer = new SentryJSONMessageDeserializer();
for (NotificationEvent event : events) {
- String dbName, tableName, oldLocation, newLocation, location;
+ String dbName, tableName, oldLocation, newLocation, location, authzObj;
+ List<String> pathTree;
switch (HCatEventMessage.EventType.valueOf(event.getEventType())) {
case CREATE_DATABASE:
SentryJSONCreateDatabaseMessage message = deserializer.getCreateDatabaseMessage(event.getMessage());
@@ -346,7 +347,15 @@ public class HMSFollower implements Runnable {
throw new SentryInvalidInputException("Could not process Create database event. Event: " + event.toString(), e);
}
}
- //TODO: HDFSPlugin.addPath(dbName, location)
+ // addPath into Sentry DB. Skip update if encounter malformed path.
+ authzObj = dbName;
+ pathTree = getPath(location);
+ if (pathTree != null) {
+ DeltaTransactionBlock deltaTransactionBlock = addPaths(authzObj, pathTree);
+ Set<String> paths = Sets.newHashSet();
+ paths.add(PathsUpdate.getPath(pathTree));
+ sentryStore.updateAuthzPathsMapping(authzObj, paths, deltaTransactionBlock);
+ }
break;
case DROP_DATABASE:
SentryJSONDropDatabaseMessage dropDatabaseMessage = deserializer.getDropDatabaseMessage(event.getMessage());
@@ -384,7 +393,15 @@ public class HMSFollower implements Runnable {
throw new SentryInvalidInputException("Could not process Create table event. Event: " + event.toString(), e);
}
}
- //TODO: HDFSPlugin.deletePath(dbName, location)
+ // addPath into Sentry DB. Skip update if encounter malformed path.
+ authzObj = dbName + "." + tableName;
+ pathTree = getPath(location);
+ if (pathTree != null) {
+ DeltaTransactionBlock deltaTransactionBlock = addPaths(authzObj, pathTree);
+ Set<String> paths = Sets.newHashSet();
+ paths.add(PathsUpdate.getPath(pathTree));
+ sentryStore.updateAuthzPathsMapping(authzObj, paths, deltaTransactionBlock);
+ }
break;
case DROP_TABLE:
SentryJSONDropTableMessage dropTableMessage = deserializer.getDropTableMessage(event.getMessage());
@@ -522,4 +539,31 @@ public class HMSFollower implements Runnable {
}
return authzObj == null ? null : authzObj.toLowerCase();
}
+
+ private DeltaTransactionBlock addPaths(String authzObj, List<String> pathTree) {
+ LOGGER.debug("#### HMS Path Update ["
+ + "OP : addPath, "
+ + "authzObj : " + authzObj.toLowerCase() + ", "
+ + "path : " + pathTree.toString() + "]");
+ PathsUpdate update = new PathsUpdate();
+ update.newPathChange(authzObj.toLowerCase()).addToAddPaths(pathTree);
+ return new DeltaTransactionBlock(update);
+ }
+
+ /**
+ * Get path tree from a given path. It throws SentryMalformedPathException
+ * if encountered malformed path.
+ *
+ * @param path
+ * @return the path tree.
+ */
+ private List<String> getPath(String path) {
+ try {
+ return PathsUpdate.parsePath(path);
+ } catch (SentryMalformedPathException e) {
+ LOGGER.error("Unexpected path = " + path);
+ e.printStackTrace();
+ return null;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/sentry/blob/96e1d9a1/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java
index a26f996..a2d48a8 100644
--- a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java
+++ b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java
@@ -39,6 +39,7 @@ import org.apache.sentry.hdfs.PermissionsUpdate;
import org.apache.sentry.hdfs.Updateable;
import org.apache.sentry.hdfs.service.thrift.TPrivilegeChanges;
import org.apache.sentry.hdfs.service.thrift.TRoleChanges;
+import org.apache.sentry.provider.db.service.model.MAuthzPathsMapping;
import org.apache.sentry.provider.db.service.model.MSentryPermChange;
import org.apache.sentry.provider.db.service.model.MSentryPrivilege;
import org.apache.sentry.provider.db.service.model.MSentryRole;
@@ -2200,6 +2201,16 @@ public class TestSentryStore extends org.junit.Assert {
assertEquals(Sets.newHashSet("/user/hive/warehouse/db1.db/table1"), pathImage.get("db1.table1"));
}
+ @Test
+ public void testUpdateAuthzPathsMapping() throws Exception {
+ sentryStore.updateAuthzPathsMapping("db1.table", Sets.newHashSet("/user/hive/warehouse/db1.db/table1"));
+ sentryStore.updateAuthzPathsMapping("db1.table", Sets.newHashSet("/user/hive/warehouse/db1.db/table2"));
+
+ MAuthzPathsMapping mAuthzPathsMapping = sentryStore.getMAuthzPathsMapping("db1.table");
+ assertEquals(Sets.newHashSet("/user/hive/warehouse/db1.db/table1", "/user/hive/warehouse/db1.db/table2"),
+ mAuthzPathsMapping.getPaths());
+ }
+
public void testQueryParamBuilder() {
SentryStore.QueryParamBuilder paramBuilder;
paramBuilder = new SentryStore.QueryParamBuilder();