You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sentry.apache.org by ka...@apache.org on 2018/06/20 12:27:52 UTC
sentry git commit: SENTRY-2275: Grant and revoke owner privileges
based on HMS updates(client-side). (Kalyan Kumar kalvagadda,
reviewed-by Sergio Pena)
Repository: sentry
Updated Branches:
refs/heads/master bee44c28a -> 86c12d64e
SENTRY-2275: Grant and revoke owner privileges based on HMS updates(client-side). (Kalyan Kumar kalvagadda, reviewed-by Sergio Pena)
Project: http://git-wip-us.apache.org/repos/asf/sentry/repo
Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/86c12d64
Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/86c12d64
Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/86c12d64
Branch: refs/heads/master
Commit: 86c12d64ee47d2644a7bcda7277bedcae98c8aba
Parents: bee44c2
Author: Kalyan Kumar Kalvagadda <kk...@cloudera.com>
Authored: Wed Jun 20 07:22:36 2018 -0500
Committer: Kalyan Kumar Kalvagadda <kk...@cloudera.com>
Committed: Wed Jun 20 07:22:36 2018 -0500
----------------------------------------------------------------------
.../binding/metastore/SentryHmsEvent.java | 221 ++++++++++++
...rySyncHMSNotificationsPostEventListener.java | 174 ++++++----
...rySyncHMSNotificationsPostEventListener.java | 337 +++++++++++++++++--
.../thrift/SentryPolicyServiceClient.java | 9 +
.../SentryPolicyServiceClientDefaultImpl.java | 11 +
5 files changed, 662 insertions(+), 90 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sentry/blob/86c12d64/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentryHmsEvent.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentryHmsEvent.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentryHmsEvent.java
new file mode 100644
index 0000000..42be3c3
--- /dev/null
+++ b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentryHmsEvent.java
@@ -0,0 +1,221 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sentry.binding.metastore;
+
+import com.google.common.collect.ImmutableMap;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.hive.metastore.MetaStoreEventListenerConstants;
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.PrincipalType;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.events.CreateDatabaseEvent;
+import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent;
+import org.apache.hadoop.hive.metastore.events.CreateTableEvent;
+import org.apache.hadoop.hive.metastore.events.DropTableEvent;
+import org.apache.hadoop.hive.metastore.events.AlterTableEvent;
+import org.apache.hadoop.hive.metastore.events.ListenerEvent;
+import org.apache.hadoop.hive.metastore.messaging.EventMessage.EventType;
+import org.apache.sentry.api.service.thrift.TSentryAuthorizable;
+import org.apache.sentry.api.service.thrift.TSentryHmsEventNotification;
+import org.apache.sentry.api.service.thrift.TSentryObjectOwnerType;
+
+import java.util.Map;
+
+/**
+ * Sentry HMS Event holds the information from the HMS Event that sentry is interested in
+ */
+class SentryHmsEvent {
+ // eventId is used to synchronize the event. If synchronization is not needed it is initialized to 0.
+ private long eventId;
+ private final EventType eventType;
+ private String ownerName;
+ private TSentryObjectOwnerType ownerType;
+ private TSentryAuthorizable authorizable;
+ private final Boolean isMetastoreTransactionActive;
+
+ private static final Map<PrincipalType, TSentryObjectOwnerType> mapOwnerType = ImmutableMap.of(
+ PrincipalType.ROLE, TSentryObjectOwnerType.ROLE,
+ PrincipalType.USER, TSentryObjectOwnerType.USER
+ );
+
+ /**
+ * Construct SentryHmsEvent from ListenerEvent and event Type
+ *
+ * event and transaction information is initialized.
+ * @param event ListenerEvent
+ * @param type EventType
+ */
+ private SentryHmsEvent(ListenerEvent event, EventType type) {
+ String isActive = event.getParameters().getOrDefault(
+ MetaStoreEventListenerConstants.HIVE_METASTORE_TRANSACTION_ACTIVE, null);
+ isMetastoreTransactionActive = (isActive != null) && Boolean.valueOf(isActive);
+ eventId = getEventId(event);
+ eventType = type;
+ }
+
+ /**
+ * Construct SentryHmsEvent from CreateTableEvent
+ *
+ * event, transaction, owner and authorizable info is initialized from event.
+ * @param event CreateTableEvent
+ */
+ public SentryHmsEvent(CreateTableEvent event) {
+ this(event, EventType.CREATE_TABLE);
+ setOwnerInfo(event.getTable());
+ setAuthorizable(event.getTable());
+ }
+
+ /**
+ * Construct SentryHmsEvent from DropTableEvent
+ *
+ * event, transaction, owner and authorizable info is initialized from event.
+ * @param event DropTableEvent
+ */
+ public SentryHmsEvent(DropTableEvent event) {
+ this(event, EventType.DROP_TABLE);
+ setOwnerInfo(event.getTable());
+ setAuthorizable(event.getTable());
+ }
+
+ /**
+ * Construct SentryHmsEvent from AlterTableEvent
+ *
+ * event, transaction, owner and authorizable info is initialized from event.
+ * @param event AlterTableEvent
+ */
+ public SentryHmsEvent(AlterTableEvent event) {
+ this(event, EventType.ALTER_TABLE);
+ if(!StringUtils.equals(event.getOldTable().getOwner(), event.getNewTable().getOwner())) {
+ // Owner Changed.
+ setOwnerInfo(event.getNewTable());
+ }
+ setAuthorizable(event.getNewTable());
+ }
+
+ /**
+ * Construct SentryHmsEvent from CreateDatabaseEvent
+ *
+ * event, transaction, owner and authorizable info is initialized from event.
+ * @param event CreateDatabaseEvent
+ */
+ public SentryHmsEvent(CreateDatabaseEvent event) {
+ this(event, EventType.CREATE_DATABASE);
+ setOwnerInfo(event.getDatabase());
+ setAuthorizable(event.getDatabase());
+ }
+
+ /**
+ * Construct SentryHmsEvent from DropDatabaseEvent
+ *
+ * event, transaction, owner and authorizable info is initialized from event.
+ * @param event DropDatabaseEvent
+ */
+ public SentryHmsEvent(DropDatabaseEvent event) {
+ this(event, EventType.DROP_DATABASE);
+ setOwnerInfo(event.getDatabase());
+ setAuthorizable(event.getDatabase());
+ }
+
+ public EventType getEventType() {
+ return eventType;
+ }
+
+ public long getEventId() {
+ return eventId;
+ }
+
+ private void setOwnerInfo(Table table) {
+ ownerName = (table != null) ? table.getOwner() : null;
+ // Hive 2.3.2 currently support owner type. Assuming user as the type for now.
+ // TODO once sentry dependency is changed to a hive version that suppots user type for table this
+ // hard coding should be rempved.
+ ownerType = TSentryObjectOwnerType.USER;
+ }
+
+ private void setOwnerInfo(Database database) {
+ ownerName = (database != null) ? database.getOwnerName() : null;
+ ownerType = (database != null) ?
+ getTSentryHmsObjectOwnerType(database.getOwnerType()) : null;
+ }
+
+ private void setAuthorizable(Table table) {
+ if (authorizable == null) {
+ authorizable = new TSentryAuthorizable();
+ }
+ authorizable.setDb((table != null) ? table.getDbName() : null);
+ authorizable.setTable((table != null) ? table.getTableName() : null);
+ }
+
+ private void setAuthorizable(Database database) {
+ if (authorizable == null) {
+ authorizable = new TSentryAuthorizable();
+ }
+ authorizable.setDb((database != null) ? database.getName() : null);
+ }
+
+ /**
+ * Updates the event_id
+ * @param eventId event id
+ */
+ public void setEventId(long eventId) {
+ this.eventId = eventId;
+ }
+
+ /**
+ * Constructs notification message that is sent to sentry server.
+ *
+ * @return notification event.
+ */
+ public TSentryHmsEventNotification getHmsEventNotification() {
+ TSentryHmsEventNotification updateAndSyncIDRequest = new TSentryHmsEventNotification();
+ updateAndSyncIDRequest.setOwnerName(ownerName);
+ updateAndSyncIDRequest.setOwnerType(ownerType);
+ updateAndSyncIDRequest.setAuthorizable(authorizable);
+ updateAndSyncIDRequest.setId(eventId);
+ updateAndSyncIDRequest.setEventType(eventType.toString());
+ return updateAndSyncIDRequest;
+ }
+
+ /**
+ * Converts Principle to Owner Type defined by sentry.
+ *
+ * @param principalType Hive Principle Type
+ * @return TSentryObjectOwnerType if the input is valid else null
+ */
+ private TSentryObjectOwnerType getTSentryHmsObjectOwnerType(PrincipalType principalType) {
+ return mapOwnerType.get(principalType);
+ }
+
+ /**
+ * Gets event-id from Event
+ *
+ * @param event HMS Event
+ * @return returns the eventId extracted from Event OR -1 if the eventId is not found the event provided.
+ */
+ private long getEventId(ListenerEvent event) {
+ return Long.parseLong(event.getParameters().getOrDefault(
+ MetaStoreEventListenerConstants.DB_NOTIFICATION_EVENT_ID_KEY_NAME, "-1"));
+ }
+
+ /**
+ * @return True if the HMS is calling this notification in an active transaction; False otherwise
+ */
+ public Boolean isMetastoreTransactionActive() {
+ return isMetastoreTransactionActive;
+ }
+}
http://git-wip-us.apache.org/repos/asf/sentry/blob/86c12d64/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentrySyncHMSNotificationsPostEventListener.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentrySyncHMSNotificationsPostEventListener.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentrySyncHMSNotificationsPostEventListener.java
index ccb60ff..f7d1b07 100644
--- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentrySyncHMSNotificationsPostEventListener.java
+++ b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentrySyncHMSNotificationsPostEventListener.java
@@ -18,11 +18,14 @@
package org.apache.sentry.binding.metastore;
import com.google.common.annotations.VisibleForTesting;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.MetaStoreEventListener;
-import org.apache.hadoop.hive.metastore.MetaStoreEventListenerConstants;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.events.AddPartitionEvent;
import org.apache.hadoop.hive.metastore.events.AlterPartitionEvent;
@@ -33,15 +36,13 @@ import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent;
import org.apache.hadoop.hive.metastore.events.DropPartitionEvent;
import org.apache.hadoop.hive.metastore.events.DropTableEvent;
import org.apache.hadoop.hive.metastore.events.ListenerEvent;
+import org.apache.hadoop.hive.metastore.messaging.EventMessage.EventType;
import org.apache.sentry.binding.hive.conf.HiveAuthzConf;
import org.apache.sentry.api.service.thrift.SentryPolicyServiceClient;
import org.apache.sentry.service.thrift.SentryServiceClientFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicLong;
-
/**
* This HMS post-event listener is used only to synchronize with HMS notifications on the Sentry server
* whenever a DDL event happens on the Hive metastore.
@@ -84,19 +85,48 @@ public class SentrySyncHMSNotificationsPostEventListener extends MetaStoreEventL
authzConf = HiveAuthzConf.getAuthzConf((HiveConf) config);
}
+ /**
+ * Notify sentry server when new table is created
+ *
+ * @param tableEvent Create table Event
+ * @throws MetaException
+ */
@Override
public void onCreateTable(CreateTableEvent tableEvent) throws MetaException {
- syncNotificationEvents(tableEvent, "onCreateTable");
+ // Failure event, Need not be notified.
+ if (failedEvent(tableEvent, EventType.CREATE_TABLE)) {
+ return;
+ }
+ SentryHmsEvent event = new SentryHmsEvent(tableEvent);
+ notifyHmsEvent(event);
}
+ /**
+ * Notify sentry server when table is dropped
+ *
+ * @param tableEvent Drop table event
+ * @throws MetaException
+ */
@Override
public void onDropTable(DropTableEvent tableEvent) throws MetaException {
- syncNotificationEvents(tableEvent, "onDropTable");
+ // Failure event, Need not be notified.
+ if (failedEvent(tableEvent, EventType.DROP_TABLE)) {
+ return;
+ }
+ SentryHmsEvent event = new SentryHmsEvent(tableEvent);
+ notifyHmsEvent(event);
}
+ /**
+ * Notify sentry server when when table is altered.
+ * Owner information is updated in the request only when there is owner change.
+ * Sentry is not notified when neither rename happened nor owner is changed
+ *
+ * @param tableEvent Alter table event
+ * @throws MetaException When both the owner change and rename is seen.
+ */
@Override
public void onAlterTable(AlterTableEvent tableEvent) throws MetaException {
-
if (tableEvent == null) {
return;
}
@@ -111,16 +141,20 @@ public class SentrySyncHMSNotificationsPostEventListener extends MetaStoreEventL
if (newTable == null) {
return;
}
+ // Failure event, Need not be notified.
+ if (failedEvent(tableEvent, EventType.ALTER_TABLE)) {
+ return;
+ }
- String oldDbName = oldTable.getDbName();
- String newDbName = newTable.getDbName();
- String oldTableName = oldTable.getTableName();
- String newTableName = newTable.getTableName();
-
- if (!newDbName.equalsIgnoreCase(oldDbName) || !newTableName.equalsIgnoreCase(oldTableName)) {
- // make sure sentry gets the table rename event
- syncNotificationEvents(tableEvent, "onAlterTable");
+ if(StringUtils.equals(oldTable.getOwner(), newTable.getOwner()) &&
+ StringUtils.equalsIgnoreCase(oldTable.getDbName(), newTable.getDbName()) &&
+ StringUtils.equalsIgnoreCase(oldTable.getTableName(), newTable.getTableName())) {
+ // nothing to notify, neither rename happened nor owner is changed
+ return;
}
+
+ SentryHmsEvent event = new SentryHmsEvent(tableEvent);
+ notifyHmsEvent(event);
}
@Override
@@ -138,76 +172,73 @@ public class SentrySyncHMSNotificationsPostEventListener extends MetaStoreEventL
// no-op
}
+ /**
+ * Notify sentry server when new database is created
+ *
+ * @param dbEvent Create database event
+ * @throws MetaException
+ */
@Override
public void onCreateDatabase(CreateDatabaseEvent dbEvent) throws MetaException {
- syncNotificationEvents(dbEvent, "onCreateDatabase");
+ // Failure event, Need not be notified.
+ if (failedEvent(dbEvent, EventType.CREATE_DATABASE)) {
+ return;
+ }
+ SentryHmsEvent event = new SentryHmsEvent(dbEvent);
+ notifyHmsEvent(event);
}
+ /**
+ * Notify sentry server when database is dropped
+ *
+ * @param dbEvent Drop database event.
+ * @throws MetaException
+ */
@Override
public void onDropDatabase(DropDatabaseEvent dbEvent) throws MetaException {
- syncNotificationEvents(dbEvent, "onDropDatabase");
+ // Failure event, Need not be notified.
+ if (failedEvent(dbEvent, EventType.DROP_DATABASE)) {
+ return;
+ }
+ SentryHmsEvent event = new SentryHmsEvent(dbEvent);
+ notifyHmsEvent(event);
}
/**
- * It requests the Sentry server the synchronization of recent notification events.
+ * Notifies sentry server about the HMS Event and related metadata.
*
- * After the sync call, the latest processed ID will be stored for future reference to avoid
- * syncing an ID that was already processed.
- *
- * @param event An event that contains a DB_NOTIFICATION_EVENT_ID_KEY_NAME value to request.
+ * @param event Sentry HMS event.
*/
- private void syncNotificationEvents(ListenerEvent event, String eventName) {
- // Do not sync notifications if the event has failed.
- if (failedEvent(event, eventName)) {
- return;
- }
-
- Map<String, String> eventParameters = event.getParameters();
- if (!eventParameters.containsKey(MetaStoreEventListenerConstants.DB_NOTIFICATION_EVENT_ID_KEY_NAME)) {
- return;
- }
-
+ private void notifyHmsEvent(SentryHmsEvent event ) {
/* If the HMS is running in an active transaction, then we do not want to sync with Sentry
* because the desired eventId is not available for Sentry yet, and Sentry may block the HMS
- * forever or until a read time-out happens. */
- if (isMetastoreTransactionActive(eventParameters)) {
+ * forever or until a read time-out happens.
+ * */
+ if(event.isMetastoreTransactionActive()) {
return;
}
- long eventId =
- Long.parseLong(eventParameters.get(MetaStoreEventListenerConstants.DB_NOTIFICATION_EVENT_ID_KEY_NAME));
-
- // This check is only for performance reasons to avoid calling the sync thrift call if the Sentry server
- // already processed the requested eventId.
- if (eventId <= latestProcessedId.get()) {
- return;
+ if (!shouldSyncEvent(event)) {
+ event.setEventId(0L);
}
- try(SentryPolicyServiceClient sentryClient = this.getSentryServiceClient()) {
- LOGGER.debug("Starting Sentry/HMS notifications sync for {} (id: {})", eventName, eventId);
- long sentryLatestProcessedId = sentryClient.syncNotifications(eventId);
- LOGGER.debug("Finished Sentry/HMS notifications sync for {} (id: {})", eventName, eventId);
+ try (SentryPolicyServiceClient sentryClient = this.getSentryServiceClient()) {
+ LOGGER.debug("Notifying sentry about Notification for {} (id: {})", event.getEventType(),
+ event.getEventId());
+ long sentryLatestProcessedId = sentryClient.notifyHmsNotification(event.getHmsEventNotification());
+ LOGGER.debug("Finished Notifying sentry about Notification for {} (id: {})", event.getEventType(),
+ event.getEventId());
LOGGER.debug("Latest processed event ID returned by the Sentry server: {}", sentryLatestProcessedId);
-
updateProcessedId(sentryLatestProcessedId);
} catch (Exception e) {
// This error is only logged. There is no need to throw an error to Hive because HMS sync is called
// after the notification is already generated by Hive (as post-event).
- LOGGER.error("Failed to sync requested HMS notifications up to the event ID: " + eventId, e);
+ LOGGER.error("Encountered failure while notifying notification for {} (id: {})",
+ event.getEventType(), event.getEventId(), e);
}
}
/**
- * @return True if the HMS is calling this notification in an active transaction; False otherwise
- */
- private boolean isMetastoreTransactionActive(Map<String, String> parameters) {
- String transactionActive =
- parameters.get(MetaStoreEventListenerConstants.HIVE_METASTORE_TRANSACTION_ACTIVE);
-
- return transactionActive != null && Boolean.valueOf(transactionActive);
- }
-
- /**
* Updates the latest processed ID, if and only if eventId is bigger. This keeps the contract that
* {@link #latestProcessedId} may only increase.
*
@@ -223,7 +254,7 @@ public class SentrySyncHMSNotificationsPostEventListener extends MetaStoreEventL
/**
* Sets the sentry client object (for testing purposes only)
- *
+ * <p>
* It may be set by unit-tests as a mock object and used to verify that the client methods
* were called correctly (see TestSentrySyncHMSNotificationsPostEventListener).
*/
@@ -245,13 +276,34 @@ public class SentrySyncHMSNotificationsPostEventListener extends MetaStoreEventL
}
}
- private boolean failedEvent(ListenerEvent event, String eventName) {
+ private boolean failedEvent(ListenerEvent event, EventType eventType) {
if (!event.getStatus()) {
LOGGER.debug("Skip HMS synchronization request with the Sentry server for {} " +
- "{} since the operation failed. \n", eventName, event);
+ "{} since the operation failed. \n", eventType.toString(), event);
return true;
}
return false;
}
+
+ /**
+ * Performs checks to make sure if the event should be synced.
+ *
+ * @param event SentryHmsEvent
+ * @return False: if Event should not be synced, True otherwise.
+ */
+ private boolean shouldSyncEvent(SentryHmsEvent event) {
+
+ // Sync need not be performed, Event id is not updated in the event.
+ if(event.getEventId() < 0) {
+ return false;
+ }
+
+ // This check is only for performance reasons to avoid calling the sync thrift call if the Sentry
+ // server already processed the requested eventId.
+ if (event.getEventId() <= latestProcessedId.get()) {
+ return false;
+ }
+ return true;
+ }
}
http://git-wip-us.apache.org/repos/asf/sentry/blob/86c12d64/sentry-binding/sentry-binding-hive/src/test/java/org/apache/sentry/binding/metastore/TestSentrySyncHMSNotificationsPostEventListener.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive/src/test/java/org/apache/sentry/binding/metastore/TestSentrySyncHMSNotificationsPostEventListener.java b/sentry-binding/sentry-binding-hive/src/test/java/org/apache/sentry/binding/metastore/TestSentrySyncHMSNotificationsPostEventListener.java
index fc1c3d5..8e79cac 100644
--- a/sentry-binding/sentry-binding-hive/src/test/java/org/apache/sentry/binding/metastore/TestSentrySyncHMSNotificationsPostEventListener.java
+++ b/sentry-binding/sentry-binding-hive/src/test/java/org/apache/sentry/binding/metastore/TestSentrySyncHMSNotificationsPostEventListener.java
@@ -20,11 +20,20 @@ package org.apache.sentry.binding.metastore;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.MetaStoreEventListenerConstants;
import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.events.AlterTableEvent;
import org.apache.hadoop.hive.metastore.events.CreateDatabaseEvent;
import org.apache.hadoop.hive.metastore.events.CreateTableEvent;
import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent;
import org.apache.hadoop.hive.metastore.events.DropTableEvent;
import org.apache.hadoop.hive.metastore.events.ListenerEvent;
+import org.apache.hadoop.hive.metastore.messaging.EventMessage;
+import org.apache.hadoop.hive.metastore.messaging.EventMessage.EventType;
+import org.apache.hadoop.hive.metastore.api.PrincipalType;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.sentry.api.service.thrift.TSentryAuthorizable;
+import org.apache.sentry.api.service.thrift.TSentryHmsEventNotification;
+import org.apache.sentry.api.service.thrift.TSentryObjectOwnerType;
import org.apache.sentry.binding.hive.conf.HiveAuthzConf;
import org.apache.sentry.core.common.exception.SentryUserException;
import org.apache.sentry.api.service.thrift.SentryPolicyServiceClient;
@@ -47,6 +56,12 @@ public class TestSentrySyncHMSNotificationsPostEventListener {
private static final boolean SUCCESSFUL_STATUS = true;
private static final boolean EVENT_ID_SET = true;
private static final boolean EVENT_ID_UNSET = false;
+ private static final String DBNAME = "db1";
+ private static final String TABLENAME = "table1";
+ private static final String TABLENAME_NEW = "table_new";
+ private static final String OWNER = "owner1";
+ private static final String OWNER_NEW = "owner_new";
+
private SentrySyncHMSNotificationsPostEventListener eventListener;
private SentryPolicyServiceClient mockSentryClient;
@@ -83,23 +98,45 @@ public class TestSentrySyncHMSNotificationsPostEventListener {
}
@Test
- public void testEventsWithoutAnEventIdDoNotSyncNotifications() throws MetaException {
- callAllEventsThatSynchronize(SUCCESSFUL_STATUS, EVENT_ID_UNSET);
- Mockito.verifyZeroInteractions(mockSentryClient);
- }
-
- @Test
public void testSuccessfulEventsWithAnEventIdSyncNotifications() throws Exception {
- long latestEventId = callAllEventsThatSynchronize(SUCCESSFUL_STATUS, EVENT_ID_SET);
+ int eventId = 1;
+
+ callAllEventsThatSynchronize(EventType.CREATE_DATABASE, SUCCESSFUL_STATUS, eventId++);
+ TSentryHmsEventNotification notification = new TSentryHmsEventNotification();
+ notification.setAuthorizable(new TSentryAuthorizable());
+
+ notification.setId(eventId - 1);
+ notification.setEventType(EventType.CREATE_DATABASE.toString());
+ Mockito.verify(
+ mockSentryClient, Mockito.times(1)
+ ).notifyHmsNotification(notification);
+
+ callAllEventsThatSynchronize(EventType.DROP_DATABASE, SUCCESSFUL_STATUS, eventId++);
+ notification.setId(eventId - 1);
+ notification.setEventType(EventType.DROP_DATABASE.toString());
+ Mockito.verify(
+ mockSentryClient, Mockito.times(1)
+ ).notifyHmsNotification(notification);
+
+ callAllEventsThatSynchronize(EventType.CREATE_TABLE, SUCCESSFUL_STATUS, eventId++);
+ notification.setId(eventId - 1);
+ notification.setEventType(EventType.CREATE_TABLE.toString());
+ notification.setOwnerType(TSentryObjectOwnerType.USER);
+ Mockito.verify(
+ mockSentryClient, Mockito.times(1)
+ ).notifyHmsNotification(notification);
+
+ long latestEventId = callAllEventsThatSynchronize(EventType.DROP_TABLE, SUCCESSFUL_STATUS, eventId++);
+ notification.setId(eventId - 1);
+ notification.setEventType(EventType.DROP_TABLE.toString());
+ notification.setOwnerType(TSentryObjectOwnerType.USER);
+ Mockito.verify(
+ mockSentryClient, Mockito.times(1)
+ ).notifyHmsNotification(notification);
- for (int i=1; i<=latestEventId; i++) {
- Mockito.verify(
- mockSentryClient, Mockito.times(1)
- ).syncNotifications(i);
- }
Mockito.verify(
- mockSentryClient, Mockito.times((int)latestEventId)
+ mockSentryClient, Mockito.times((int) latestEventId)
).close();
Mockito.verifyNoMoreInteractions(mockSentryClient);
@@ -110,52 +147,294 @@ public class TestSentrySyncHMSNotificationsPostEventListener {
Mockito.doAnswer(new Answer<Long>() {
@Override
public Long answer(InvocationOnMock invocation) throws Throwable {
- Long id = (Long)invocation.getArguments()[0];
- return id + 1;
+ TSentryHmsEventNotification notification = (TSentryHmsEventNotification) invocation.getArguments()[0];
+ return notification.getId() + 1;
}
- }).when(mockSentryClient).syncNotifications(Mockito.anyLong());
+ }).when(mockSentryClient).notifyHmsNotification(Mockito.anyObject());
long latestEventId = callAllEventsThatSynchronize(SUCCESSFUL_STATUS, EVENT_ID_SET);
-
- for (int i=1; i<=latestEventId; i+=2) {
- Mockito.verify(
- mockSentryClient, Mockito.times(1)
- ).syncNotifications(i);
- }
+ verifyInvocations();
Mockito.verify(
- mockSentryClient, Mockito.times((int)latestEventId / 2)
+ mockSentryClient, Mockito.times((int) latestEventId)
).close();
+ }
- Mockito.verifyNoMoreInteractions(mockSentryClient);
+ @Test
+ public void notificationOnTableCreate() throws Exception {
+ int eventId = 1;
+ Table tb = new Table();
+ tb.setDbName(DBNAME);
+ tb.setTableName(TABLENAME);
+ tb.setOwner(OWNER);
+ CreateTableEvent createTableEvent = new CreateTableEvent(tb, true, null);
+ setEventId(EVENT_ID_SET, createTableEvent, eventId);
+ eventListener.onCreateTable(createTableEvent);
+
+ TSentryHmsEventNotification notification = new TSentryHmsEventNotification();
+
+ notification.setAuthorizable(new TSentryAuthorizable());
+
+ notification.setId(eventId);
+ notification.setEventType(EventMessage.EventType.CREATE_TABLE.toString());
+ notification.setOwnerName(OWNER);
+ notification.setOwnerType(TSentryObjectOwnerType.USER);
+ notification.getAuthorizable().setDb(DBNAME);
+ notification.getAuthorizable().setTable(TABLENAME);
+ Mockito.verify(
+ mockSentryClient, Mockito.times(1)
+ ).notifyHmsNotification(notification);
+ }
+
+ @Test
+ public void notificationOnTableDrop() throws Exception {
+ int eventId = 1;
+ Table tb = new Table();
+ tb.setDbName(DBNAME);
+ tb.setTableName(TABLENAME);
+ tb.setOwner(OWNER);
+ DropTableEvent dropTableEvent = new DropTableEvent(tb, true, true, null);
+ setEventId(EVENT_ID_SET, dropTableEvent, eventId);
+ eventListener.onDropTable(dropTableEvent);
+
+ TSentryHmsEventNotification notification = new TSentryHmsEventNotification();
+
+ notification.setAuthorizable(new TSentryAuthorizable());
+
+ notification.setId(eventId);
+ notification.setEventType(EventMessage.EventType.DROP_TABLE.toString());
+ notification.setOwnerName(OWNER);
+ notification.setOwnerType(TSentryObjectOwnerType.USER);
+ notification.getAuthorizable().setDb(DBNAME);
+ notification.getAuthorizable().setTable(TABLENAME);
+ Mockito.verify(
+ mockSentryClient, Mockito.times(1)
+ ).notifyHmsNotification(notification);
+ }
+
+ @Test
+ public void notificationOnDatabaseCreate() throws Exception {
+ int eventId = 1;
+ Database db = new Database();
+ db.setName(DBNAME);
+ db.setOwnerName(OWNER);
+ db.setOwnerType(PrincipalType.USER);
+ CreateDatabaseEvent createDatabaseEvent = new CreateDatabaseEvent(db, true, null);
+ setEventId(EVENT_ID_SET, createDatabaseEvent, eventId);
+ eventListener.onCreateDatabase(createDatabaseEvent);
+
+ TSentryHmsEventNotification notification = new TSentryHmsEventNotification();
+
+ notification.setAuthorizable(new TSentryAuthorizable());
+
+ notification.setId(eventId);
+ notification.setEventType(EventMessage.EventType.CREATE_DATABASE.toString());
+ notification.setOwnerName(OWNER);
+ notification.setOwnerType(TSentryObjectOwnerType.USER);
+ notification.getAuthorizable().setDb(DBNAME);
+ Mockito.verify(
+ mockSentryClient, Mockito.times(1)
+ ).notifyHmsNotification(notification);
+ }
+
+ @Test
+ public void notificationOnDatabaseDrop() throws Exception {
+ int eventId = 1;
+ Database db = new Database();
+ db.setName(DBNAME);
+ db.setOwnerName(OWNER);
+ db.setOwnerType(PrincipalType.USER);
+ DropDatabaseEvent dropDatabaseEvent = new DropDatabaseEvent(db, true, null);
+ setEventId(EVENT_ID_SET, dropDatabaseEvent, eventId);
+ eventListener.onDropDatabase(dropDatabaseEvent);
+
+ TSentryHmsEventNotification notification = new TSentryHmsEventNotification();
+
+ notification.setAuthorizable(new TSentryAuthorizable());
+
+ notification.setId(eventId);
+ notification.setEventType(EventType.DROP_DATABASE.toString());
+ notification.setOwnerName(OWNER);
+ notification.setOwnerType(TSentryObjectOwnerType.USER);
+ notification.getAuthorizable().setDb(DBNAME);
+ Mockito.verify(
+ mockSentryClient, Mockito.times(1)
+ ).notifyHmsNotification(notification);
+ }
+
+
+ @Test
+ public void notificationOnAlterTableOwnerChange() throws Exception {
+ int eventId = 1;
+ Table old_tb = new Table();
+ old_tb.setDbName(DBNAME);
+ old_tb.setTableName(TABLENAME);
+ old_tb.setOwner(OWNER);
+
+ Table new_tb = new Table();
+ new_tb.setDbName(DBNAME);
+ new_tb.setTableName(TABLENAME);
+ new_tb.setOwner(OWNER_NEW);
+
+ AlterTableEvent alterTableEvent = new AlterTableEvent(old_tb, new_tb, true, null);
+ setEventId(EVENT_ID_SET, alterTableEvent, eventId);
+ eventListener.onAlterTable(alterTableEvent);
+
+ TSentryHmsEventNotification notification = new TSentryHmsEventNotification();
+
+ notification.setAuthorizable(new TSentryAuthorizable());
+
+ notification.setId(eventId);
+ notification.setEventType(EventType.ALTER_TABLE.toString());
+ notification.setOwnerName(OWNER_NEW);
+ notification.setOwnerType(TSentryObjectOwnerType.USER);
+ notification.getAuthorizable().setDb(DBNAME);
+ notification.getAuthorizable().setTable(TABLENAME);
+
+ Mockito.verify(
+ mockSentryClient, Mockito.times(1)
+ ).notifyHmsNotification(notification);
+ }
+
+ @Test
+ public void notificationOnAlterTableRename() throws Exception {
+ int eventId = 1;
+ Table old_tb = new Table();
+ old_tb.setDbName(DBNAME);
+ old_tb.setTableName(TABLENAME);
+ old_tb.setOwner(OWNER);
+
+ Table new_tb = new Table();
+ new_tb.setDbName(DBNAME);
+ new_tb.setTableName(TABLENAME_NEW);
+ new_tb.setOwner(OWNER);
+
+ AlterTableEvent alterTableEvent = new AlterTableEvent(old_tb, new_tb, true, null);
+ setEventId(EVENT_ID_SET, alterTableEvent, eventId);
+ eventListener.onAlterTable(alterTableEvent);
+
+ TSentryHmsEventNotification notification = new TSentryHmsEventNotification();
+
+ notification.setAuthorizable(new TSentryAuthorizable());
+
+ notification.setId(eventId);
+ notification.setEventType(EventType.ALTER_TABLE.toString());
+ notification.getAuthorizable().setDb(DBNAME);
+ notification.getAuthorizable().setTable(TABLENAME_NEW);
+
+ Mockito.verify(
+ mockSentryClient, Mockito.times(1)
+ ).notifyHmsNotification(notification);
+ }
+
+ @Test
+ public void notificationOnAlterTableNoRenameAndOwnerChange() throws Exception {
+ int eventId = 1;
+ Table old_tb = new Table();
+ old_tb.setDbName(DBNAME);
+ old_tb.setTableName(TABLENAME);
+ old_tb.setOwner(OWNER);
+
+ Table new_tb = new Table();
+ new_tb.setDbName(DBNAME);
+ new_tb.setTableName(TABLENAME);
+ new_tb.setOwner(OWNER);
+
+ AlterTableEvent alterTableEvent = new AlterTableEvent(old_tb, new_tb, true, null);
+ setEventId(EVENT_ID_SET, alterTableEvent, eventId);
+ eventListener.onAlterTable(alterTableEvent);
+
+ TSentryHmsEventNotification notification = new TSentryHmsEventNotification();
+
+ notification.setAuthorizable(new TSentryAuthorizable());
+
+ notification.setId(eventId);
+ notification.setEventType(EventType.ALTER_TABLE.toString());
+ notification.getAuthorizable().setDb(DBNAME);
+ notification.getAuthorizable().setTable(TABLENAME);
+
+ Mockito.verify(
+ mockSentryClient, Mockito.times(0)
+ ).notifyHmsNotification(Mockito.anyObject());
}
private long callAllEventsThatSynchronize(boolean status, boolean eventIdSet) throws MetaException {
long eventId = 0;
- CreateDatabaseEvent createDatabaseEvent = new CreateDatabaseEvent(null, status , null);
+ CreateDatabaseEvent createDatabaseEvent = new CreateDatabaseEvent(null, status, null);
setEventId(eventIdSet, createDatabaseEvent, ++eventId);
eventListener.onCreateDatabase(createDatabaseEvent);
- DropDatabaseEvent dropDatabaseEvent = new DropDatabaseEvent(null, status , null);
+ DropDatabaseEvent dropDatabaseEvent = new DropDatabaseEvent(null, status, null);
setEventId(eventIdSet, dropDatabaseEvent, ++eventId);
eventListener.onDropDatabase(dropDatabaseEvent);
- CreateTableEvent createTableEvent = new CreateTableEvent(null, status , null);
+ CreateTableEvent createTableEvent = new CreateTableEvent(null, status, null);
setEventId(eventIdSet, createTableEvent, ++eventId);
eventListener.onCreateTable(createTableEvent);
- DropTableEvent dropTableEvent = new DropTableEvent(null, status , false, null);
+ DropTableEvent dropTableEvent = new DropTableEvent(null, status, false, null);
setEventId(eventIdSet, dropTableEvent, ++eventId);
eventListener.onDropTable(dropTableEvent);
return eventId;
}
+ private long callAllEventsThatSynchronize(EventType event, boolean status, long eventId) throws MetaException {
+ switch (event) {
+ case CREATE_DATABASE:
+ CreateDatabaseEvent createDatabaseEvent = new CreateDatabaseEvent(null, status, null);
+ setEventId(true, createDatabaseEvent, eventId);
+ eventListener.onCreateDatabase(createDatabaseEvent);
+ break;
+ case DROP_DATABASE:
+ DropDatabaseEvent dropDatabaseEvent = new DropDatabaseEvent(null, status, null);
+ setEventId(true, dropDatabaseEvent, eventId);
+ eventListener.onDropDatabase(dropDatabaseEvent);
+ break;
+ case CREATE_TABLE:
+
+ CreateTableEvent createTableEvent = new CreateTableEvent(null, status, null);
+ setEventId(true, createTableEvent, eventId);
+ eventListener.onCreateTable(createTableEvent);
+ break;
+ case DROP_TABLE:
+ DropTableEvent dropTableEvent = new DropTableEvent(null, status, false, null);
+ setEventId(true, dropTableEvent, eventId);
+ eventListener.onDropTable(dropTableEvent);
+ break;
+ default:
+ return eventId;
+ }
+ return eventId;
+ }
+
+ private void verifyInvocations() throws Exception {
+ TSentryHmsEventNotification notification = new TSentryHmsEventNotification();
+ int i = 1;
+ notification.setAuthorizable(new TSentryAuthorizable());
+
+ notification.setId(i);
+ notification.setEventType(EventMessage.EventType.CREATE_DATABASE.toString());
+ Mockito.verify(
+ mockSentryClient, Mockito.times(1)
+ ).notifyHmsNotification(notification);
+ i += 2;
+
+
+ notification.setId(i);
+ notification.setEventType(EventMessage.EventType.CREATE_TABLE.toString());
+ notification.setOwnerType(TSentryObjectOwnerType.USER);
+ Mockito.verify(
+ mockSentryClient, Mockito.times(1)
+ ).notifyHmsNotification(notification);
+ }
+
private void setEventId(boolean eventIdSet, ListenerEvent eventListener, long eventId) {
if (eventIdSet) {
eventListener.putParameter(
- MetaStoreEventListenerConstants.DB_NOTIFICATION_EVENT_ID_KEY_NAME, String.valueOf(eventId));
+ MetaStoreEventListenerConstants.DB_NOTIFICATION_EVENT_ID_KEY_NAME, String.valueOf(eventId));
}
}
+
}
http://git-wip-us.apache.org/repos/asf/sentry/blob/86c12d64/sentry-service/sentry-service-api/src/main/java/org/apache/sentry/api/service/thrift/SentryPolicyServiceClient.java
----------------------------------------------------------------------
diff --git a/sentry-service/sentry-service-api/src/main/java/org/apache/sentry/api/service/thrift/SentryPolicyServiceClient.java b/sentry-service/sentry-service-api/src/main/java/org/apache/sentry/api/service/thrift/SentryPolicyServiceClient.java
index 3950ea5..dc1d67b 100644
--- a/sentry-service/sentry-service-api/src/main/java/org/apache/sentry/api/service/thrift/SentryPolicyServiceClient.java
+++ b/sentry-service/sentry-service-api/src/main/java/org/apache/sentry/api/service/thrift/SentryPolicyServiceClient.java
@@ -292,4 +292,13 @@ public interface SentryPolicyServiceClient extends AutoCloseable {
* @return The most recent processed notification ID.
*/
long syncNotifications(long id) throws SentryUserException;
+
+ /**
+ * Notifies sentry server with the HMS Event and related metadata.
+ * @param sentryHmsEventNotification Event Notification message.
+ * @return The most recent processed notification ID.
+ */
+ long notifyHmsNotification(TSentryHmsEventNotification sentryHmsEventNotification) throws SentryUserException;
+
+
}
http://git-wip-us.apache.org/repos/asf/sentry/blob/86c12d64/sentry-service/sentry-service-api/src/main/java/org/apache/sentry/api/service/thrift/SentryPolicyServiceClientDefaultImpl.java
----------------------------------------------------------------------
diff --git a/sentry-service/sentry-service-api/src/main/java/org/apache/sentry/api/service/thrift/SentryPolicyServiceClientDefaultImpl.java b/sentry-service/sentry-service-api/src/main/java/org/apache/sentry/api/service/thrift/SentryPolicyServiceClientDefaultImpl.java
index 28d345e..5f76cb0 100644
--- a/sentry-service/sentry-service-api/src/main/java/org/apache/sentry/api/service/thrift/SentryPolicyServiceClientDefaultImpl.java
+++ b/sentry-service/sentry-service-api/src/main/java/org/apache/sentry/api/service/thrift/SentryPolicyServiceClientDefaultImpl.java
@@ -1167,4 +1167,15 @@ public class SentryPolicyServiceClientDefaultImpl implements SentryPolicyService
throw new SentryUserException(THRIFT_EXCEPTION_MESSAGE, e);
}
}
+
+ public long notifyHmsNotification(TSentryHmsEventNotification request)
+ throws SentryUserException {
+ try {
+ TSentryHmsEventNotificationResponse response = client.sentry_notify_hms_event(request);
+ Status.throwIfNotOk(response.getStatus());
+ return response.getId();
+ } catch (TException e) {
+ throw new SentryUserException(THRIFT_EXCEPTION_MESSAGE, e);
+ }
+ }
}
\ No newline at end of file