You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sentry.apache.org by ka...@apache.org on 2017/07/14 00:57:40 UTC
[4/6] sentry git commit: SENTRY-1769 Refactor HMSFollower Class
(Kalyan Kumar Kalvagadda reviewed by Vamsee Yarlagadda, Na Li,
Sergio Pena and Alexander Kolbasov)
http://git-wip-us.apache.org/repos/asf/sentry/blob/024d99de/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestHmsFollower.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestHmsFollower.java b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestHmsFollower.java
new file mode 100644
index 0000000..2095469
--- /dev/null
+++ b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestHmsFollower.java
@@ -0,0 +1,532 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sentry.service.thrift;
+
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.NotificationEvent;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hive.hcatalog.messaging.HCatEventMessage;
+import org.apache.hive.hcatalog.messaging.HCatEventMessage.EventType;
+import org.apache.sentry.binding.metastore.messaging.json.SentryJSONMessageFactory;
+import org.apache.sentry.hdfs.Updateable;
+import org.apache.sentry.provider.db.service.persistent.SentryStore;
+import org.apache.sentry.provider.db.service.thrift.TSentryAuthorizable;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import javax.security.auth.login.LoginException;
+
+public class TestHmsFollower {
+
+ private final static String hiveInstance = "server2";
+ private final static Configuration configuration = new Configuration();
+ private final SentryJSONMessageFactory messageFactory = new SentryJSONMessageFactory();
+ private final SentryStore sentryStore = Mockito.mock(SentryStore.class);
+ private static HiveSimpleConnectionFactory hiveConnectionFactory;
+
+ @BeforeClass
+ public static void setup() throws IOException, LoginException {
+ hiveConnectionFactory = new HiveSimpleConnectionFactory(configuration, new HiveConf());
+ hiveConnectionFactory.init();
+ configuration.set("sentry.hive.sync.create", "true");
+ }
+
+ /**
+ * Constructs create database event and makes sure that appropriate sentry store API's
+ * are invoke when the event is processed by hms follower.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testCreateDatabase() throws Exception {
+ String dbName = "db1";
+
+ // Create notification events
+ NotificationEvent notificationEvent = new NotificationEvent(1, 0,
+ HCatEventMessage.EventType.CREATE_DATABASE.toString(),
+ messageFactory.buildCreateDatabaseMessage(new Database(dbName, null, "hdfs:///db1", null))
+ .toString());
+ List<NotificationEvent> events = new ArrayList<>();
+ events.add(notificationEvent);
+ HmsFollower hmsFollower = new HmsFollower(configuration, sentryStore, null,
+ hiveConnectionFactory, hiveInstance);
+ hmsFollower.processNotifications(events);
+
+ TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance);
+ authorizable.setServer(hiveInstance);
+ authorizable.setDb("db1");
+
+ verify(sentryStore, times(1))
+ .dropPrivilege(authorizable, NotificationProcessor.getPermUpdatableOnDrop(authorizable));
+ }
+
+ /**
+ * Constructs drop database event and makes sure that appropriate sentry store API's
+ * are invoke when the event is processed by hms follower.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testDropDatabase() throws Exception {
+ String dbName = "db1";
+
+ // Create notification events
+ NotificationEvent notificationEvent = new NotificationEvent(1, 0,
+ HCatEventMessage.EventType.DROP_DATABASE.toString(),
+ messageFactory.buildDropDatabaseMessage(new Database(dbName, null, "hdfs:///db1", null))
+ .toString());
+ List<NotificationEvent> events = new ArrayList<>();
+ events.add(notificationEvent);
+
+ HmsFollower hmsFollower = new HmsFollower(configuration, sentryStore, null,
+ hiveConnectionFactory, hiveInstance);
+ hmsFollower.processNotifications(events);
+
+ TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance);
+ authorizable.setServer(hiveInstance);
+ authorizable.setDb("db1");
+
+ verify(sentryStore, times(1))
+ .dropPrivilege(authorizable, NotificationProcessor.getPermUpdatableOnDrop(authorizable));
+ }
+
+ /**
+ * Constructs create table event and makes sure that appropriate sentry store API's
+ * are invoke when the event is processed by hms follower.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testCreateTable() throws Exception {
+ String dbName = "db1";
+ String tableName = "table1";
+
+ // Create notification events
+ StorageDescriptor sd = new StorageDescriptor();
+ sd.setLocation("hdfs:///db1.db/table1");
+ NotificationEvent notificationEvent = new NotificationEvent(1, 0,
+ HCatEventMessage.EventType.CREATE_TABLE.toString(),
+ messageFactory.buildCreateTableMessage(
+ new Table(tableName, dbName, null, 0, 0, 0, sd, null, null, null, null, null))
+ .toString());
+ List<NotificationEvent> events = new ArrayList<>();
+ events.add(notificationEvent);
+
+ HmsFollower hmsFollower = new HmsFollower(configuration, sentryStore, null,
+ hiveConnectionFactory, hiveInstance);
+ hmsFollower.processNotifications(events);
+
+ TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance);
+ authorizable.setServer(hiveInstance);
+ authorizable.setDb("db1");
+ authorizable.setTable(tableName);
+
+ verify(sentryStore, times(1))
+ .dropPrivilege(authorizable, NotificationProcessor.getPermUpdatableOnDrop(authorizable));
+ }
+
+ /**
+ * Constructs drop table event and makes sure that appropriate sentry store API's
+ * are invoke when the event is processed by hms follower.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testDropTable() throws Exception {
+ String dbName = "db1";
+ String tableName = "table1";
+
+ // Create notification events
+ StorageDescriptor sd = new StorageDescriptor();
+ sd.setLocation("hdfs:///db1.db/table1");
+ NotificationEvent notificationEvent = new NotificationEvent(1, 0,
+ HCatEventMessage.EventType.DROP_TABLE.toString(),
+ messageFactory.buildDropTableMessage(
+ new Table(tableName, dbName, null, 0, 0, 0, sd, null, null, null, null, null))
+ .toString());
+ List<NotificationEvent> events = new ArrayList<>();
+ events.add(notificationEvent);
+
+ HmsFollower hmsFollower = new HmsFollower(configuration, sentryStore, null,
+ hiveConnectionFactory, hiveInstance);
+ hmsFollower.processNotifications(events);
+
+ TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance);
+ authorizable.setServer(hiveInstance);
+ authorizable.setDb("db1");
+ authorizable.setTable(tableName);
+
+ verify(sentryStore, times(1))
+ .dropPrivilege(authorizable, NotificationProcessor.getPermUpdatableOnDrop(authorizable));
+ }
+
+ /**
+ * Constructs rename table event and makes sure that appropriate sentry store API's
+ * are invoke when the event is processed by hms follower.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testRenameTable() throws Exception {
+ String dbName = "db1";
+ String tableName = "table1";
+
+ String newDbName = "db1";
+ String newTableName = "table2";
+
+ // Create notification events
+ StorageDescriptor sd = new StorageDescriptor();
+ sd.setLocation("hdfs:///db1.db/table1");
+ NotificationEvent notificationEvent = new NotificationEvent(1, 0,
+ HCatEventMessage.EventType.ALTER_TABLE.toString(),
+ messageFactory.buildAlterTableMessage(
+ new Table(tableName, dbName, null, 0, 0, 0, sd, null, null, null, null, null),
+ new Table(newTableName, newDbName, null, 0, 0, 0, sd, null, null, null, null, null))
+ .toString());
+ notificationEvent.setDbName(newDbName);
+ notificationEvent.setTableName(newTableName);
+ List<NotificationEvent> events = new ArrayList<>();
+ events.add(notificationEvent);
+
+ HmsFollower hmsFollower = new HmsFollower(configuration, sentryStore, null,
+ hiveConnectionFactory, hiveInstance);
+ hmsFollower.processNotifications(events);
+
+ TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance);
+ authorizable.setServer(hiveInstance);
+ authorizable.setDb(dbName);
+ authorizable.setTable(tableName);
+
+ TSentryAuthorizable newAuthorizable = new TSentryAuthorizable(hiveInstance);
+ authorizable.setServer(hiveInstance);
+ newAuthorizable.setDb(newDbName);
+ newAuthorizable.setTable(newTableName);
+
+ verify(sentryStore, times(1)).renamePrivilege(authorizable, newAuthorizable,
+ NotificationProcessor.getPermUpdatableOnRename(authorizable, newAuthorizable));
+ }
+
+
+ @Ignore
+ /**
+ * Constructs a bunch of events and passed to processor of hms follower. One of those is alter
+ * partition event with out actually changing anything(invalid event). Idea is to make sure that
+ * hms follower calls appropriate sentry store API's for the events processed by hms follower
+ * after processing the invalid alter partition event.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testAlterPartitionWithInvalidEvent() throws Exception {
+ String dbName = "db1";
+ String tableName1 = "table1";
+ String tableName2 = "table2";
+ long inputEventId = 1;
+ List<NotificationEvent> events = new ArrayList<>();
+ NotificationEvent notificationEvent;
+ List<FieldSchema> partCols;
+ StorageDescriptor sd;
+ Mockito.doNothing().when(sentryStore).persistLastProcessedNotificationID(Mockito.anyLong());
+ //noinspection unchecked
+ Mockito.doNothing().when(sentryStore).addAuthzPathsMapping(Mockito.anyString(),
+ Mockito.anyCollection(), Mockito.any(Updateable.Update.class));
+
+ Configuration configuration = new Configuration();
+ HmsFollower hmsFollower = new HmsFollower(configuration, sentryStore, null,
+ hiveConnectionFactory, hiveInstance);
+ // Create a table
+ sd = new StorageDescriptor();
+ sd.setLocation("hdfs://db1.db/table1");
+ partCols = new ArrayList<>();
+ partCols.add(new FieldSchema("ds", "string", ""));
+ Table table = new Table(tableName1, dbName, null, 0, 0, 0, sd, partCols, null, null, null,
+ null);
+ notificationEvent = new NotificationEvent(inputEventId, 0,
+ HCatEventMessage.EventType.CREATE_TABLE.toString(),
+ messageFactory.buildCreateTableMessage(table).toString());
+ notificationEvent.setDbName(dbName);
+ notificationEvent.setTableName(tableName1);
+ events.add(notificationEvent);
+ inputEventId += 1;
+ // Process the notification
+ hmsFollower.processNotifications(events);
+ // Make sure that addAuthzPathsMapping was invoked once to handle CREATE_TABLE notification
+ // and persistLastProcessedNotificationID was not invoked.
+ //noinspection unchecked
+ verify(sentryStore, times(1)).addAuthzPathsMapping(Mockito.anyString(),
+ Mockito.anyCollection(), Mockito.any(Updateable.Update.class));
+ verify(sentryStore, times(0)).persistLastProcessedNotificationID(Mockito.anyLong());
+ reset(sentryStore);
+ events.clear();
+
+ // Create a partition
+ List<Partition> partitions = new ArrayList<>();
+ StorageDescriptor invalidSd = new StorageDescriptor();
+ invalidSd.setLocation(null);
+ Partition partition = new Partition(Collections.singletonList("today"), dbName, tableName1,
+ 0, 0, sd, null);
+ partitions.add(partition);
+ notificationEvent = new NotificationEvent(inputEventId, 0, EventType.ADD_PARTITION.toString(),
+ messageFactory.buildAddPartitionMessage(table, partitions).toString());
+ notificationEvent.setDbName(dbName);
+ notificationEvent.setTableName(tableName1);
+ events.add(notificationEvent);
+ inputEventId += 1;
+ //Process the notification
+ hmsFollower.processNotifications(events);
+ // Make sure that addAuthzPathsMapping was invoked once to handle ADD_PARTITION notification
+ // and persistLastProcessedNotificationID was not invoked.
+ //noinspection unchecked
+ verify(sentryStore, times(1)).addAuthzPathsMapping(Mockito.anyString(),
+ Mockito.anyCollection(), Mockito.any(Updateable.Update.class));
+ verify(sentryStore, times(0)).persistLastProcessedNotificationID(Mockito.anyLong());
+ reset(sentryStore);
+ events.clear();
+
+ // Create a alter notification with out actually changing anything.
+ // This is an invalid event and should be processed by sentry store.
+ // Event Id should be explicitly persisted using persistLastProcessedNotificationID
+ notificationEvent = new NotificationEvent(inputEventId, 0, EventType.ALTER_PARTITION.toString(),
+ messageFactory.buildAlterPartitionMessage(partition, partition).toString());
+ notificationEvent.setDbName(dbName);
+ notificationEvent.setTableName(tableName1);
+ events.add(notificationEvent);
+ inputEventId += 1;
+ // Process the notification
+ hmsFollower.processNotifications(events);
+ // Make sure that persistLastProcessedNotificationID is invoked explicitly.
+ verify(sentryStore, times(1)).persistLastProcessedNotificationID(inputEventId - 1);
+ reset(sentryStore);
+ events.clear();
+
+ // Create a alter notification with some actual change.
+ sd = new StorageDescriptor();
+ sd.setLocation("hdfs://user/hive/warehouse/db1.db/table1");
+ Partition updatedPartition = new Partition(partition);
+ updatedPartition.setSd(sd);
+ notificationEvent = new NotificationEvent(inputEventId, 0, EventType.ALTER_PARTITION.toString(),
+ messageFactory.buildAlterPartitionMessage(partition, updatedPartition).toString());
+ notificationEvent.setDbName(dbName);
+ notificationEvent.setTableName(tableName1);
+ events.add(notificationEvent);
+ inputEventId += 1;
+ // Process the notification
+ hmsFollower.processNotifications(events);
+ // Make sure that updateAuthzPathsMapping was invoked once to handle ALTER_PARTITION
+ // notification and persistLastProcessedNotificationID was not invoked.
+ verify(sentryStore, times(1)).updateAuthzPathsMapping(Mockito.anyString(),
+ Mockito.anyString(), Mockito.anyString(), Mockito.any(Updateable.Update.class));
+ verify(sentryStore, times(0)).persistLastProcessedNotificationID(inputEventId - 1);
+ reset(sentryStore);
+ events.clear();
+
+ // Create a table
+ sd = new StorageDescriptor();
+ sd.setLocation("hdfs://db1.db/table2");
+ partCols = new ArrayList<>();
+ partCols.add(new FieldSchema("ds", "string", ""));
+ Table table1 = new Table(tableName2, dbName, null, 0, 0, 0, sd, partCols, null, null, null,
+ null);
+ notificationEvent = new NotificationEvent(inputEventId, 0,
+ HCatEventMessage.EventType.CREATE_TABLE.toString(),
+ messageFactory.buildCreateTableMessage(table1).toString());
+ notificationEvent.setDbName(dbName);
+ notificationEvent.setTableName(tableName2);
+ events.add(notificationEvent);
+ // Process the notification
+ hmsFollower.processNotifications(events);
+ // Make sure that addAuthzPathsMapping was invoked once to handle CREATE_TABLE notification
+ // and persistLastProcessedNotificationID was not invoked.
+ //noinspection unchecked
+ verify(sentryStore, times(1)).addAuthzPathsMapping(Mockito.anyString(),
+ Mockito.anyCollection(), Mockito.any(Updateable.Update.class));
+ verify(sentryStore, times(0)).persistLastProcessedNotificationID(Mockito.anyLong());
+ }
+
+ /**
+ * Constructs a bunch of events and passed to processor of hms follower. One of those is alter
+ * table event with out actually changing anything(invalid event). Idea is to make sure that
+ * hms follower calls appropriate sentry store API's for the events processed by hms follower
+ * after processing the invalid alter table event.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testAlterTableWithInvalidEvent() throws Exception {
+ String dbName = "db1";
+ String tableName1 = "table1";
+ String tableName2 = "table2";
+ long inputEventId = 1;
+ List<NotificationEvent> events = new ArrayList<>();
+ NotificationEvent notificationEvent;
+ List<FieldSchema> partCols;
+ StorageDescriptor sd;
+ Mockito.doNothing().when(sentryStore).persistLastProcessedNotificationID(Mockito.anyLong());
+ //noinspection unchecked
+ Mockito.doNothing().when(sentryStore).addAuthzPathsMapping(Mockito.anyString(),
+ Mockito.anyCollection(), Mockito.any(Updateable.Update.class));
+
+ Configuration configuration = new Configuration();
+ HmsFollower hmsFollower = new HmsFollower(configuration, sentryStore, null,
+ hiveConnectionFactory, hiveInstance);
+
+ // Create a table
+ sd = new StorageDescriptor();
+ sd.setLocation("hdfs://db1.db/table1");
+ partCols = new ArrayList<>();
+ partCols.add(new FieldSchema("ds", "string", ""));
+ Table table = new Table(tableName1, dbName, null, 0, 0, 0, sd, partCols, null, null, null,
+ null);
+ notificationEvent = new NotificationEvent(inputEventId, 0,
+ HCatEventMessage.EventType.CREATE_TABLE.toString(),
+ messageFactory.buildCreateTableMessage(table).toString());
+ notificationEvent.setDbName(dbName);
+ notificationEvent.setTableName(tableName1);
+ events.add(notificationEvent);
+ inputEventId += 1;
+ // Process the notification
+ hmsFollower.processNotifications(events);
+ // Make sure that addAuthzPathsMapping was invoked once to handle CREATE_TABLE notification
+ // and persistLastProcessedNotificationID was not invoked.
+ //noinspection unchecked
+ verify(sentryStore, times(1)).addAuthzPathsMapping(Mockito.anyString(),
+ Mockito.anyCollection(), Mockito.any(Updateable.Update.class));
+ verify(sentryStore, times(0)).persistLastProcessedNotificationID(Mockito.anyLong());
+ reset(sentryStore);
+ events.clear();
+
+ // Create alter table notification with out actually changing anything.
+ // This notification should not be processed by sentry server
+ // Notification should be persisted explicitly
+ notificationEvent = new NotificationEvent(1, 0,
+ HCatEventMessage.EventType.ALTER_TABLE.toString(),
+ messageFactory.buildAlterTableMessage(
+ new Table(tableName1, dbName, null, 0, 0, 0, sd, null, null, null, null, null),
+ new Table(tableName1, dbName, null, 0, 0, 0, sd, null, null, null, null, null))
+ .toString());
+ notificationEvent.setDbName(dbName);
+ notificationEvent.setTableName(tableName1);
+ events = new ArrayList<>();
+ events.add(notificationEvent);
+ inputEventId += 1;
+ // Process the notification
+ hmsFollower.processNotifications(events);
+ // Make sure that renameAuthzObj and deleteAuthzPathsMapping were not invoked
+ // to handle CREATE_TABLE notification
+ // and persistLastProcessedNotificationID is explicitly invoked
+ verify(sentryStore, times(0)).renameAuthzObj(Mockito.anyString(), Mockito.anyString(),
+ Mockito.any(Updateable.Update.class));
+ //noinspection unchecked
+ verify(sentryStore, times(0)).deleteAuthzPathsMapping(Mockito.anyString(),
+ Mockito.anyCollection(), Mockito.any(Updateable.Update.class));
+ verify(sentryStore, times(1)).persistLastProcessedNotificationID(Mockito.anyLong());
+ reset(sentryStore);
+ events.clear();
+
+ // Create a table
+ sd = new StorageDescriptor();
+ sd.setLocation("hdfs://db1.db/table2");
+ partCols = new ArrayList<>();
+ partCols.add(new FieldSchema("ds", "string", ""));
+ Table table1 = new Table(tableName2, dbName, null, 0, 0, 0, sd, partCols, null, null, null,
+ null);
+ notificationEvent = new NotificationEvent(inputEventId, 0,
+ HCatEventMessage.EventType.CREATE_TABLE.toString(),
+ messageFactory.buildCreateTableMessage(table1).toString());
+ notificationEvent.setDbName(dbName);
+ notificationEvent.setTableName(tableName2);
+ events.add(notificationEvent);
+ // Process the notification
+ hmsFollower.processNotifications(events);
+ // Make sure that addAuthzPathsMapping was invoked once to handle CREATE_TABLE notification
+ // and persistLastProcessedNotificationID was not invoked.
+ //noinspection unchecked
+ verify(sentryStore, times(1)).addAuthzPathsMapping(Mockito.anyString(),
+ Mockito.anyCollection(), Mockito.any(Updateable.Update.class));
+ verify(sentryStore, times(0)).persistLastProcessedNotificationID(Mockito.anyLong());
+ }
+
+ /**
+ * Constructs a two events and passed to processor of hms follower. First one being create table
+ * event with location information(Invalid Event). Idea is to make sure that hms follower calls
+ * appropriate sentry store API's for the event processed by hms follower after processing the
+ * invalid create table event.
+ *
+ * @throws Exception
+ */
+ public void testCreateTableAfterInvalidEvent() throws Exception {
+ String dbName = "db1";
+ String tableName = "table1";
+ long inputEventId = 1;
+
+ Mockito.doNothing().when(sentryStore).persistLastProcessedNotificationID(Mockito.anyLong());
+ //noinspection unchecked
+ Mockito.doNothing().when(sentryStore)
+ .addAuthzPathsMapping(Mockito.anyString(), Mockito.anyCollection(),
+ Mockito.any(Updateable.Update.class));
+
+ // Create invalid notification event. The location of the storage descriptor is null, which is invalid for creating table
+ StorageDescriptor invalidSd = new StorageDescriptor();
+ invalidSd.setLocation(null);
+ NotificationEvent invalidNotificationEvent = new NotificationEvent(inputEventId, 0,
+ HCatEventMessage.EventType.CREATE_TABLE.toString(),
+ messageFactory.buildCreateTableMessage(
+ new Table(tableName, dbName, null, 0, 0, 0, invalidSd, null, null, null, null, null))
+ .toString());
+
+ // Create valid notification event
+ StorageDescriptor sd = new StorageDescriptor();
+ sd.setLocation("hdfs://db1.db/table1");
+ inputEventId += 1;
+ NotificationEvent notificationEvent = new NotificationEvent(inputEventId, 0,
+ HCatEventMessage.EventType.CREATE_TABLE.toString(),
+ messageFactory.buildCreateTableMessage(
+ new Table(tableName, dbName, null, 0, 0, 0, sd, null, null, null, null, null))
+ .toString());
+ List<NotificationEvent> events = new ArrayList<>();
+ events.add(invalidNotificationEvent);
+ events.add(notificationEvent);
+
+ Configuration configuration = new Configuration();
+ HmsFollower hmsFollower = new HmsFollower(configuration, sentryStore, null,
+ hiveConnectionFactory, hiveInstance);
+ hmsFollower.processNotifications(events);
+
+ // invalid event updates notification ID directly
+ verify(sentryStore, times(1)).persistLastProcessedNotificationID(inputEventId - 1);
+
+ // next valid event update path, which updates notification ID
+ //noinspection unchecked
+ verify(sentryStore, times(1)).addAuthzPathsMapping(Mockito.anyString(), Mockito.anyCollection(),
+ Mockito.any(Updateable.Update.class));
+ }
+}
http://git-wip-us.apache.org/repos/asf/sentry/blob/024d99de/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestNotificationProcessor.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestNotificationProcessor.java b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestNotificationProcessor.java
new file mode 100644
index 0000000..c6c9448
--- /dev/null
+++ b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestNotificationProcessor.java
@@ -0,0 +1,465 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sentry.service.thrift;
+
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.NotificationEvent;
+import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hive.hcatalog.messaging.HCatEventMessage;
+import org.apache.sentry.binding.metastore.messaging.json.SentryJSONMessageFactory;
+import org.apache.sentry.hdfs.Updateable;
+import org.apache.sentry.provider.db.service.persistent.SentryStore;
+import org.apache.sentry.provider.db.service.thrift.TSentryAuthorizable;
+import org.junit.After;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+// TODO 1. More tests should be added here.
+// TODO 2. Tests using actual sentry store where.
+@SuppressWarnings("unused")
+public class TestNotificationProcessor {
+
+ private static final SentryStore sentryStore = Mockito.mock(SentryStore.class);
+ private final static String hiveInstance = "server2";
+ private final static Configuration conf = new Configuration();
+ private final SentryJSONMessageFactory messageFactory = new SentryJSONMessageFactory();
+ private NotificationProcessor notificationProcessor;
+
+ @BeforeClass
+ public static void setup() {
+ conf.set("sentry.hive.sync.create", "true");
+ conf.set("sentry.hive.sync.drop", "true");
+ }
+
+ @After
+ public void resetConf() {
+ conf.set("sentry.hive.sync.create", "true");
+ conf.set("sentry.hive.sync.drop", "true");
+ reset(sentryStore);
+ }
+
+ @Test
+ /*
+ Makes sure that appropriate sentry store methods are invoked when create database event is
+ processed.
+
+ Also, checks the hive sync configuration.
+ */
+ public void testCreateDatabase() throws Exception {
+ long seqNum = 1;
+ String dbName = "db1";
+ String uriPrefix = "hdfs:///";
+ String location = "user/hive/warehouse";
+ NotificationEvent notificationEvent;
+ TSentryAuthorizable authorizable;
+ notificationProcessor = new NotificationProcessor(sentryStore,
+ hiveInstance, conf);
+
+ // Create notification event
+ notificationEvent = new NotificationEvent(seqNum, 0,
+ HCatEventMessage.EventType.CREATE_DATABASE.toString(),
+ messageFactory.buildCreateDatabaseMessage(new Database(dbName,
+ null, uriPrefix + location, null)).toString());
+
+ notificationProcessor.processNotificationEvent(notificationEvent);
+
+ authorizable = new TSentryAuthorizable(hiveInstance);
+ authorizable.setServer(hiveInstance);
+ authorizable.setDb("db1");
+ //noinspection unchecked
+ verify(sentryStore, times(1)).addAuthzPathsMapping(Mockito.anyString(),
+ Mockito.anyCollection(), Mockito.any(Updateable.Update.class));
+
+ verify(sentryStore, times(1)).dropPrivilege(authorizable,
+ NotificationProcessor.getPermUpdatableOnDrop(authorizable));
+ reset(sentryStore);
+
+ //Change the configuration and make sure that exiting privileges are not dropped
+ notificationProcessor.setSyncStoreOnCreate(false);
+ dbName = "db2";
+ notificationEvent = new NotificationEvent(1, 0,
+ HCatEventMessage.EventType.CREATE_DATABASE.toString(),
+ messageFactory.buildCreateDatabaseMessage(new Database(dbName,
+ null, "hdfs:///db2", null)).toString());
+
+ notificationProcessor.processNotificationEvent(notificationEvent);
+
+ authorizable = new TSentryAuthorizable(hiveInstance);
+ authorizable.setServer(hiveInstance);
+ authorizable.setDb(dbName);
+
+ //noinspection unchecked
+ verify(sentryStore, times(1)).addAuthzPathsMapping(Mockito.anyString(),
+ Mockito.anyCollection(), Mockito.any(Updateable.Update.class));
+ //making sure that privileges are not dropped
+ verify(sentryStore, times(0)).dropPrivilege(authorizable,
+ NotificationProcessor.getPermUpdatableOnDrop(authorizable));
+
+ }
+
+ @Test
+ /*
+ Makes sure that appropriate sentry store methods are invoked when drop database event is
+ processed.
+
+ Also, checks the hive sync configuration.
+ */
+ public void testDropDatabase() throws Exception {
+ String dbName = "db1";
+
+ notificationProcessor = new NotificationProcessor(sentryStore,
+ hiveInstance, conf);
+
+ // Create notification event
+ NotificationEvent notificationEvent = new NotificationEvent(1, 0,
+ HCatEventMessage.EventType.DROP_DATABASE.toString(),
+ messageFactory.buildDropDatabaseMessage(new Database(dbName, null,
+ "hdfs:///db1", null)).toString());
+
+ notificationProcessor.processNotificationEvent(notificationEvent);
+
+ TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance);
+ authorizable.setServer(hiveInstance);
+ authorizable.setDb("db1");
+
+ //noinspection unchecked
+ verify(sentryStore, times(1)).deleteAuthzPathsMapping(Mockito.anyString(),
+ Mockito.anyCollection(), Mockito.any(Updateable.Update.class));
+ verify(sentryStore, times(1)).dropPrivilege(authorizable,
+ NotificationProcessor.getPermUpdatableOnDrop(authorizable));
+ reset(sentryStore);
+
+ // Change the configuration and make sure that exiting privileges are not dropped
+ notificationProcessor.setSyncStoreOnDrop(false);
+ dbName = "db2";
+ // Create notification event
+ notificationEvent = new NotificationEvent(1, 0,
+ HCatEventMessage.EventType.DROP_DATABASE.toString(),
+ messageFactory.buildDropDatabaseMessage(new Database(dbName, null,
+ "hdfs:///db2", null)).toString());
+
+ notificationProcessor.processNotificationEvent(notificationEvent);
+
+ authorizable = new TSentryAuthorizable(hiveInstance);
+ authorizable.setServer(hiveInstance);
+ authorizable.setDb(dbName);
+
+ //noinspection unchecked
+ verify(sentryStore, times(1)).deleteAuthzPathsMapping(Mockito.anyString(),
+ Mockito.anyCollection(), Mockito.any(Updateable.Update.class));
+ verify(sentryStore, times(0)).dropPrivilege(authorizable,
+ NotificationProcessor.getPermUpdatableOnDrop(authorizable));
+ }
+
+ @Test
+ /*
+ Makes sure that appropriate sentry store methods are invoked when create table event is
+ processed.
+
+ Also, checks the hive sync configuration.
+ */
+ public void testCreateTable() throws Exception {
+ String dbName = "db1";
+ String tableName = "table1";
+
+ notificationProcessor = new NotificationProcessor(sentryStore,
+ hiveInstance, conf);
+
+ // Create notification event
+ StorageDescriptor sd = new StorageDescriptor();
+ sd.setLocation("hdfs:///db1.db/table1");
+ NotificationEvent notificationEvent =
+ new NotificationEvent(1, 0, HCatEventMessage.EventType.CREATE_TABLE.toString(),
+ messageFactory.buildCreateTableMessage(new Table(tableName,
+ dbName, null, 0, 0, 0, sd, null, null, null, null, null)).toString());
+
+ notificationProcessor.processNotificationEvent(notificationEvent);
+
+ TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance);
+ authorizable.setServer(hiveInstance);
+ authorizable.setDb("db1");
+ authorizable.setTable(tableName);
+
+ //noinspection unchecked
+ verify(sentryStore, times(1)).addAuthzPathsMapping(Mockito.anyString(),
+ Mockito.anyCollection(), Mockito.any(Updateable.Update.class));
+
+ verify(sentryStore, times(1)).dropPrivilege(authorizable,
+ NotificationProcessor.getPermUpdatableOnDrop(authorizable));
+ reset(sentryStore);
+
+ // Change the configuration and make sure that existing privileges are not dropped
+ notificationProcessor.setSyncStoreOnCreate(false);
+
+ // Create notification event
+ dbName = "db2";
+ tableName = "table2";
+ sd = new StorageDescriptor();
+ sd.setLocation("hdfs:///db1.db/table2");
+ notificationEvent =
+ new NotificationEvent(1, 0, HCatEventMessage.EventType.CREATE_TABLE.toString(),
+ messageFactory.buildCreateTableMessage(new Table(tableName,
+ dbName, null, 0, 0, 0, sd, null, null, null, null, null)).toString());
+
+ notificationProcessor.processNotificationEvent(notificationEvent);
+
+ authorizable = new TSentryAuthorizable(hiveInstance);
+ authorizable.setServer(hiveInstance);
+ authorizable.setDb(dbName);
+ authorizable.setTable(tableName);
+
+ //noinspection unchecked
+ verify(sentryStore, times(1)).addAuthzPathsMapping(Mockito.anyString(),
+ Mockito.anyCollection(), Mockito.any(Updateable.Update.class));
+ // Making sure that privileges are not dropped
+ verify(sentryStore, times(0)).dropPrivilege(authorizable,
+ NotificationProcessor.getPermUpdatableOnDrop(authorizable));
+ }
+
+ @Test
+ /*
+ Makes sure that appropriate sentry store methods are invoked when drop table event is
+ processed.
+
+ Also, checks the hive sync configuration.
+ */
+ public void testDropTable() throws Exception {
+ String dbName = "db1";
+ String tableName = "table1";
+
+ Configuration authConf = new Configuration();
+ notificationProcessor = new NotificationProcessor(sentryStore,
+ hiveInstance, authConf);
+
+ // Create notification event
+ StorageDescriptor sd = new StorageDescriptor();
+ sd.setLocation("hdfs:///db1.db/table1");
+ NotificationEvent notificationEvent = new NotificationEvent(1, 0,
+ HCatEventMessage.EventType.DROP_TABLE.toString(),
+ messageFactory.buildDropTableMessage(new Table(tableName,
+ dbName, null, 0, 0, 0, sd, null, null, null, null, null)).toString());
+
+ notificationProcessor.processNotificationEvent(notificationEvent);
+
+ TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance);
+ authorizable.setServer(hiveInstance);
+ authorizable.setDb("db1");
+ authorizable.setTable(tableName);
+
+ verify(sentryStore, times(1)).deleteAllAuthzPathsMapping(Mockito.anyString(),
+ Mockito.any(Updateable.Update.class));
+
+ verify(sentryStore, times(1)).dropPrivilege(authorizable,
+ NotificationProcessor.getPermUpdatableOnDrop(authorizable));
+ }
+
+ @Test
+ /*
+ Makes sure that appropriate sentry store methods are invoked when alter tables event is
+ processed.
+ */
+ public void testAlterTable() throws Exception {
+ String dbName = "db1";
+ String tableName = "table1";
+
+ String newDbName = "db1";
+ String newTableName = "table2";
+
+ Configuration authConf = new Configuration();
+ notificationProcessor = new NotificationProcessor(sentryStore,
+ hiveInstance, authConf);
+
+ // Create notification event
+ StorageDescriptor sd = new StorageDescriptor();
+ sd.setLocation("hdfs:///db1.db/table1");
+ NotificationEvent notificationEvent = new NotificationEvent(1, 0,
+ HCatEventMessage.EventType.ALTER_TABLE.toString(),
+ messageFactory.buildAlterTableMessage(
+ new Table(tableName, dbName, null, 0, 0, 0, sd, null, null, null, null, null),
+ new Table(newTableName, newDbName, null, 0, 0, 0, sd, null, null, null, null, null))
+ .toString());
+ notificationEvent.setDbName(newDbName);
+ notificationEvent.setTableName(newTableName);
+
+ notificationProcessor.processNotificationEvent(notificationEvent);
+
+ TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance);
+ authorizable.setServer(hiveInstance);
+ authorizable.setDb(dbName);
+ authorizable.setTable(tableName);
+
+ TSentryAuthorizable newAuthorizable = new TSentryAuthorizable(hiveInstance);
+ authorizable.setServer(hiveInstance);
+ newAuthorizable.setDb(newDbName);
+ newAuthorizable.setTable(newTableName);
+
+ verify(sentryStore, times(1)).renameAuthzObj(Mockito.anyString(), Mockito.anyString(),
+ Mockito.any(Updateable.Update.class));
+
+ verify(sentryStore, times(1)).renamePrivilege(authorizable, newAuthorizable,
+ NotificationProcessor.getPermUpdatableOnRename(authorizable, newAuthorizable));
+ }
+
+ @Test
+ /*
+ Makes sure that appropriate sentry store methods are invoked when alter tables event is
+ processed.
+ */
+ public void testRenameTableWithLocationUpdate() throws Exception {
+ String dbName = "db1";
+ String tableName = "table1";
+
+ String newDbName = "db1";
+ String newTableName = "table2";
+
+ Configuration authConf = new Configuration();
+ notificationProcessor = new NotificationProcessor(sentryStore,
+ hiveInstance, authConf);
+
+ // Create notification event
+ StorageDescriptor sd = new StorageDescriptor();
+ sd.setLocation("hdfs:///db1.db/table1");
+ StorageDescriptor new_sd = new StorageDescriptor();
+ new_sd.setLocation("hdfs:///db1.db/table2");
+ NotificationEvent notificationEvent = new NotificationEvent(1, 0,
+ HCatEventMessage.EventType.ALTER_TABLE.toString(),
+ messageFactory.buildAlterTableMessage(
+ new Table(tableName, dbName, null, 0, 0, 0, sd, null, null, null, null, null),
+ new Table(newTableName, newDbName, null, 0, 0, 0, new_sd, null, null, null, null, null))
+ .toString());
+ notificationEvent.setDbName(newDbName);
+ notificationEvent.setTableName(newTableName);
+
+ notificationProcessor.processNotificationEvent(notificationEvent);
+
+ TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance);
+ authorizable.setServer(hiveInstance);
+ authorizable.setDb(dbName);
+ authorizable.setTable(tableName);
+
+ TSentryAuthorizable newAuthorizable = new TSentryAuthorizable(hiveInstance);
+ authorizable.setServer(hiveInstance);
+ newAuthorizable.setDb(newDbName);
+ newAuthorizable.setTable(newTableName);
+
+ verify(sentryStore, times(1)).renameAuthzPathsMapping(Mockito.anyString(), Mockito.anyString(),
+ Mockito.anyString(), Mockito.anyString(), Mockito.any(Updateable.Update.class));
+
+ verify(sentryStore, times(1)).renamePrivilege(authorizable, newAuthorizable,
+ NotificationProcessor.getPermUpdatableOnRename(authorizable, newAuthorizable));
+ }
+
+ @Test
+ /*
+ Test to made sure that sentry store is not invoked when invalid alter table event is
+ processed.
+ */
+ public void testAlterTableWithInvalidEvent() throws Exception {
+ String dbName = "db1";
+ String tableName1 = "table1";
+ String tableName2 = "table2";
+ long inputEventId = 1;
+ NotificationEvent notificationEvent;
+ List<FieldSchema> partCols;
+ StorageDescriptor sd;
+ Mockito.doNothing().when(sentryStore).persistLastProcessedNotificationID(Mockito.anyLong());
+ //noinspection unchecked
+ Mockito.doNothing().when(sentryStore).addAuthzPathsMapping(Mockito.anyString(),
+ Mockito.anyCollection(), Mockito.any(Updateable.Update.class));
+
+ Configuration authConf = new Configuration();
+ notificationProcessor = new NotificationProcessor(sentryStore,
+ hiveInstance, authConf);
+
+ // Create a table
+ sd = new StorageDescriptor();
+ sd.setLocation("hdfs://db1.db/table1");
+ partCols = new ArrayList<>();
+ partCols.add(new FieldSchema("ds", "string", ""));
+ Table table = new Table(tableName1, dbName, null, 0, 0, 0, sd, partCols,
+ null, null, null, null);
+ notificationEvent = new NotificationEvent(inputEventId, 0,
+ HCatEventMessage.EventType.CREATE_TABLE.toString(),
+ messageFactory.buildCreateTableMessage(table).toString());
+ notificationEvent.setDbName(dbName);
+ notificationEvent.setTableName(tableName1);
+ inputEventId += 1;
+ // Process the notification
+ notificationProcessor.processNotificationEvent(notificationEvent);
+ // Make sure that addAuthzPathsMapping was invoked once to handle CREATE_TABLE notification
+ // and persistLastProcessedNotificationID was not invoked.
+ //noinspection unchecked
+ verify(sentryStore, times(1)).addAuthzPathsMapping(Mockito.anyString(),
+ Mockito.anyCollection(), Mockito.any(Updateable.Update.class));
+ reset(sentryStore);
+
+ // Create alter table notification with out actually changing anything.
+ // This notification should not be processed by sentry server
+ // Notification should be persisted explicitly
+ notificationEvent = new NotificationEvent(1, 0,
+ HCatEventMessage.EventType.ALTER_TABLE.toString(),
+ messageFactory.buildAlterTableMessage(
+ new Table(tableName1, dbName, null, 0, 0, 0, sd, null, null, null, null, null),
+ new Table(tableName1, dbName, null, 0, 0, 0, sd, null,
+ null, null, null, null)).toString());
+ notificationEvent.setDbName(dbName);
+ notificationEvent.setTableName(tableName1);
+ inputEventId += 1;
+ // Process the notification
+ notificationProcessor.processNotificationEvent(notificationEvent);
+ // Make sure that renameAuthzObj and deleteAuthzPathsMapping were not invoked
+ // to handle CREATE_TABLE notification
+ // and persistLastProcessedNotificationID is explicitly invoked
+ verify(sentryStore, times(0)).renameAuthzObj(Mockito.anyString(), Mockito.anyString(),
+ Mockito.any(Updateable.Update.class));
+ //noinspection unchecked
+ verify(sentryStore, times(0)).deleteAuthzPathsMapping(Mockito.anyString(),
+ Mockito.anyCollection(), Mockito.any(Updateable.Update.class));
+ reset(sentryStore);
+
+ // Create a table
+ sd = new StorageDescriptor();
+ sd.setLocation("hdfs://db1.db/table2");
+ partCols = new ArrayList<>();
+ partCols.add(new FieldSchema("ds", "string", ""));
+ Table table1 = new Table(tableName2, dbName, null, 0, 0, 0, sd,
+ partCols, null, null, null, null);
+ notificationEvent = new NotificationEvent(inputEventId, 0,
+ HCatEventMessage.EventType.CREATE_TABLE.toString(),
+ messageFactory.buildCreateTableMessage(table1).toString());
+ notificationEvent.setDbName(dbName);
+ notificationEvent.setTableName(tableName2);
+ // Process the notification
+ notificationProcessor.processNotificationEvent(notificationEvent);
+ // Make sure that addAuthzPathsMapping was invoked once to handle CREATE_TABLE notification
+ // and persistLastProcessedNotificationID was not invoked.
+ //noinspection unchecked
+ verify(sentryStore, times(1)).addAuthzPathsMapping(Mockito.anyString(),
+ Mockito.anyCollection(), Mockito.any(Updateable.Update.class));
+ }
+}
http://git-wip-us.apache.org/repos/asf/sentry/blob/024d99de/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestSentryHmsClient.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestSentryHmsClient.java b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestSentryHmsClient.java
new file mode 100644
index 0000000..3cc6541
--- /dev/null
+++ b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestSentryHmsClient.java
@@ -0,0 +1,470 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.service.thrift;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Random;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.IMetaStoreClient.NotificationFilter;
+import org.apache.hadoop.hive.metastore.api.CurrentNotificationEventId;
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.NotificationEvent;
+import org.apache.hadoop.hive.metastore.api.NotificationEventResponse;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hive.hcatalog.messaging.HCatEventMessage;
+import org.apache.sentry.binding.metastore.messaging.json.SentryJSONMessageFactory;
+import org.apache.sentry.provider.db.service.persistent.PathsImage;
+import org.apache.thrift.TException;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.mockito.Mockito;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+import javax.security.auth.login.LoginException;
+
+/**
+ * Test mocks HiveMetaStoreClient class and tests SentryHmsClient.
+ */
+public class TestSentryHmsClient {
+
+ private static final Configuration conf = new Configuration();
+ private static final SentryJSONMessageFactory messageFactory = new SentryJSONMessageFactory();
+ private static SentryHmsClient client;
+ private static MockHMSClientFactory hiveConnectionFactory;
+
+ /**
+ * Create mock database with the given name
+ *
+ * @param name Database name
+ * @return Mock database object
+ */
+ private static Database makeDb(String name) {
+ Database db = Mockito.mock(Database.class);
+ Mockito.when(db.getName()).thenReturn(name);
+ Mockito.when(db.getLocationUri()).thenReturn("hdfs:///" + name);
+ return db;
+ }
+
+ /**
+ * Create mock table
+ *
+ * @param dbName db for this table
+ * @param tableName name of the table
+ * @return mock table object
+ */
+ private static Table makeTable(String dbName, String tableName) {
+ Table table = Mockito.mock(Table.class);
+ Mockito.when(table.getDbName()).thenReturn(dbName);
+ Mockito.when(table.getTableName()).thenReturn(tableName);
+ StorageDescriptor sd = Mockito.mock(StorageDescriptor.class);
+ Mockito.when(sd.getLocation()).thenReturn(
+ String.format("hdfs:///%s/%s", dbName, tableName));
+ Mockito.when(table.getSd()).thenReturn(sd);
+ return table;
+ }
+
+ /**
+ * Create mock partition
+ *
+ * @param dbName database for this partition
+ * @param tableName table for this partition
+ * @param partName partition name
+ * @return mock partition object
+ */
+ private static Partition makePartition(String dbName, String tableName, String partName) {
+ Partition partition = Mockito.mock(Partition.class);
+ StorageDescriptor sd = Mockito.mock(StorageDescriptor.class);
+ Mockito.when(sd.getLocation()).thenReturn(
+ String.format("hdfs:///%s/%s/%s", dbName, tableName, partName));
+ Mockito.when(partition.getSd()).thenReturn(sd);
+ return partition;
+ }
+
+ /**
+ * Creates create database notification
+ *
+ * @return NotificationEvent
+ */
+ private static NotificationEvent getCreateDatabaseNotification(long id) {
+ Random rand = new Random();
+ int n = rand.nextInt(100) + 1;
+ String dbName = "db" + n;
+ return new NotificationEvent(id, 0, HCatEventMessage.EventType.CREATE_DATABASE.toString(),
+ messageFactory
+ .buildCreateDatabaseMessage(new Database(dbName, null, "hdfs:///" + dbName, null))
+ .toString());
+ }
+
+ /**
+ * Creates drop database notification
+ *
+ * @return NotificationEvent
+ */
+ private static NotificationEvent getDropDatabaseNotification(long id) {
+ Random rand = new Random();
+ int n = rand.nextInt(100) + 1;
+ String dbName = "db" + n;
+ return new NotificationEvent(id, 0, HCatEventMessage.EventType.DROP_DATABASE.toString(),
+ messageFactory
+ .buildDropDatabaseMessage(new Database(dbName, null, "hdfs:///" + dbName, null))
+ .toString());
+ }
+
+ @BeforeClass
+ static public void initialize() throws IOException, LoginException {
+ hiveConnectionFactory = new MockHMSClientFactory();
+ client = new SentryHmsClient(conf, (HiveConnectionFactory)hiveConnectionFactory);
+ }
+
+ /**
+ * Creating snapshot when SentryHmsClient is not connected to HMS
+ */
+ @Test
+ public void testSnapshotCreationWithOutClientConnected() throws Exception {
+ // Make sure that client is not connected
+ Assert.assertFalse(client.isConnected());
+ PathsImage snapshotInfo = client.getFullSnapshot();
+ Assert.assertTrue(snapshotInfo.getPathImage().isEmpty());
+ }
+
+ /**
+ * Creating snapshot when HMS doesn't have any data
+ */
+ @Test
+ public void testSnapshotCreationWithNoHmsData() throws Exception {
+ MockClient mockClient = new MockClient(new HiveSnapshot(), 1);
+ client.setClient(mockClient.client);
+ // Make sure that client is connected
+ Assert.assertTrue(client.isConnected());
+ PathsImage snapshotInfo = client.getFullSnapshot();
+ Assert.assertTrue(snapshotInfo.getPathImage().isEmpty());
+ }
+
+ /**
+ * Creating a snapshot when there is data but there are updates to HMS data mean while
+ */
+ @Test
+ public void testSnapshotCreationWhenDataIsActivelyUpdated() throws Exception {
+ HiveTable tab21 = new HiveTable("tab21");
+ HiveTable tab31 = new HiveTable("tab31").add("part311").add("part312");
+ HiveDb db3 = new HiveDb("db3", Lists.newArrayList(tab31));
+ HiveDb db2 = new HiveDb("db2", Lists.newArrayList(tab21));
+ HiveDb db1 = new HiveDb("db1");
+ HiveSnapshot snap = new HiveSnapshot().add(db1).add(db2).add(db3);
+ final MockClient mockClient = new MockClient(snap, 1);
+
+ client.setClient(mockClient.client);
+ hiveConnectionFactory.setClient(mockClient);
+ // Make sure that client is connected
+ Assert.assertTrue(client.isConnected());
+ PathsImage snapshotInfo = client.getFullSnapshot();
+ // Make sure that snapshot is not empty
+ Assert.assertTrue(!snapshotInfo.getPathImage().isEmpty());
+
+ Mockito.when(mockClient.client.getCurrentNotificationEventId()).
+ thenAnswer(new Answer<CurrentNotificationEventId>() {
+ @Override
+ public CurrentNotificationEventId answer(InvocationOnMock invocation)
+ throws Throwable {
+ return new CurrentNotificationEventId(mockClient.incrementNotificationEventId());
+ }
+
+ });
+
+ snapshotInfo = client.getFullSnapshot();
+ Assert.assertTrue(snapshotInfo.getPathImage().isEmpty());
+ }
+
+ /**
+ * Creating a snapshot when there is data in HMS.
+ */
+ @Test
+ public void testSnapshotCreationSuccess() throws Exception {
+ HiveTable tab21 = new HiveTable("tab21");
+ HiveTable tab31 = new HiveTable("tab31");
+ HiveDb db3 = new HiveDb("db3", Lists.newArrayList(tab31));
+ HiveDb db2 = new HiveDb("db2", Lists.newArrayList(tab21));
+ HiveDb db1 = new HiveDb("db1");
+ HiveSnapshot snap = new HiveSnapshot().add(db1).add(db2).add(db3);
+ MockClient mockClient = new MockClient(snap, 1);
+ Mockito.when(mockClient.client.getCurrentNotificationEventId()).
+ thenReturn(new CurrentNotificationEventId(mockClient.eventId));
+ client.setClient(mockClient.client);
+ hiveConnectionFactory.setClient(mockClient);
+ // Make sure that client is connected
+ Assert.assertTrue(client.isConnected());
+
+ PathsImage snapshotInfo = client.getFullSnapshot();
+ Assert.assertEquals(5, snapshotInfo.getPathImage().size());
+ Assert.assertEquals(Sets.newHashSet("db1"), snapshotInfo.getPathImage().get("db1"));
+ Assert.assertEquals(Sets.newHashSet("db2"), snapshotInfo.getPathImage().get("db2"));
+ Assert.assertEquals(Sets.newHashSet("db3"), snapshotInfo.getPathImage().get("db3"));
+ Assert.assertEquals(Sets.newHashSet("db2/tab21"),
+ snapshotInfo.getPathImage().get("db2.tab21"));
+ Assert.assertEquals(Sets.newHashSet("db3/tab31"), snapshotInfo.getPathImage().get("db3.tab31"));
+ Assert.assertEquals(snapshotInfo.getId(), mockClient.eventId);
+
+ }
+
+ /**
+ * Test scenario when there is no HMS connection
+ * Getting new notifications
+ */
+ @Test
+ public void testGetNewNotificationsWithOutClientConnected() throws Exception {
+ HiveTable tab21 = new HiveTable("tab21");
+ HiveTable tab31 = new HiveTable("tab31");
+ HiveDb db3 = new HiveDb("db3", Lists.newArrayList(tab31));
+ HiveDb db2 = new HiveDb("db2", Lists.newArrayList(tab21));
+ HiveDb db1 = new HiveDb("db1");
+ client.setClient(null);
+ HiveSnapshot snap = new HiveSnapshot().add(db1).add(db2).add(db3);
+ MockClient mockClient = new MockClient(snap, 100);
+ Mockito.when(mockClient.client.getCurrentNotificationEventId()).
+ thenReturn(new CurrentNotificationEventId(mockClient.eventId));
+ // Make sure that client is not connected
+ Assert.assertTrue(!client.isConnected());
+ Collection<NotificationEvent> events = client.getNotifications(100);
+ Assert.assertTrue(events.isEmpty());
+
+ }
+
+ /**
+ * Test scenario where there are no notifications
+ * Getting new notifications
+ */
+ @Test
+ public void testGetNewNotificationsWithNoHmsUpdates() throws Exception {
+ HiveTable tab21 = new HiveTable("tab21");
+ HiveTable tab31 = new HiveTable("tab31");
+ HiveDb db3 = new HiveDb("db3", Lists.newArrayList(tab31));
+ HiveDb db2 = new HiveDb("db2", Lists.newArrayList(tab21));
+ HiveDb db1 = new HiveDb("db1");
+ HiveSnapshot snap = new HiveSnapshot().add(db1).add(db2).add(db3);
+ MockClient mockClient = new MockClient(snap, 100);
+ Mockito.when(mockClient.client.getCurrentNotificationEventId()).
+ thenReturn(new CurrentNotificationEventId(mockClient.eventId));
+ client.setClient(mockClient.client);
+ hiveConnectionFactory.setClient(mockClient);
+ // Make sure that client is connected
+ Assert.assertTrue(client.isConnected());
+ Collection<NotificationEvent> events = client.getNotifications(100);
+ Assert.assertTrue(events.isEmpty());
+ }
+
+ /**
+ * Test scenario where there are notifications
+ * Getting new notifications
+ */
+ @Test
+ public void testGetNewNotificationsSuccess() throws Exception {
+ final MockClient mockClient = new MockClient(new HiveSnapshot(), 100);
+ client.setClient(mockClient.client);
+ hiveConnectionFactory.setClient(mockClient);
+ // Make sure that client is connected
+ Assert.assertTrue(client.isConnected());
+
+ Mockito.when(mockClient.client.getCurrentNotificationEventId()).
+ thenAnswer(new Answer<CurrentNotificationEventId>() {
+ @Override
+ public CurrentNotificationEventId answer(InvocationOnMock invocation)
+ throws Throwable {
+ return new CurrentNotificationEventId(mockClient.incrementNotificationEventId());
+ }
+ });
+ Mockito.when(mockClient.client.getNextNotification(Mockito.anyLong(), Mockito.anyInt(),
+ Mockito.any(NotificationFilter.class))).
+ thenAnswer(new Answer<NotificationEventResponse>() {
+ @Override
+ public NotificationEventResponse answer(InvocationOnMock invocation)
+ throws Throwable {
+ long id = 1;
+ List<NotificationEvent> events = new ArrayList<>();
+ events.add(getCreateDatabaseNotification(id++));
+ events.add(getDropDatabaseNotification(id++));
+ return new NotificationEventResponse(events);
+ }
+ });
+
+ Collection<NotificationEvent> events = client.getNotifications(100);
+ long id = 1;
+ for (NotificationEvent event : events) {
+ Assert.assertEquals(event.getEventId(), id++);
+ }
+ Assert.assertTrue(events.size() == 2);
+ }
+
+ /**
+ * Representation of a Hive table. A table has a name and a list of partitions.
+ */
+ private static class HiveTable {
+
+ private final String name;
+ private final List<String> partitions;
+
+ HiveTable(String name) {
+ this.name = name;
+ this.partitions = new ArrayList<>();
+ }
+
+ HiveTable add(String partition) {
+ partitions.add(partition);
+ return this;
+ }
+ }
+
+ /**
+ * Representation of a Hive database. A database has a name and a list of tables
+ */
+ private static class HiveDb {
+
+ final String name;
+ Collection<HiveTable> tables;
+
+ @SuppressWarnings("SameParameterValue")
+ HiveDb(String name) {
+ this.name = name;
+ tables = new ArrayList<>();
+ }
+
+ HiveDb(String name, Collection<HiveTable> tables) {
+ this.name = name;
+ this.tables = tables;
+ if (this.tables == null) {
+ this.tables = new ArrayList<>();
+ }
+ }
+
+ void add(HiveTable table) {
+ this.tables.add(table);
+ }
+ }
+
+ /**
+ * Representation of a full Hive snapshot. A snapshot is collection of databases
+ */
+ private static class HiveSnapshot {
+
+ final List<HiveDb> databases = new ArrayList<>();
+
+ HiveSnapshot() {
+ }
+
+ HiveSnapshot(Collection<HiveDb> dblist) {
+ if (dblist != null) {
+ databases.addAll(dblist);
+ }
+ }
+
+ HiveSnapshot add(HiveDb db) {
+ this.databases.add(db);
+ return this;
+ }
+ }
+
+ /**
+ * Mock for HMSClientFactory
+ */
+ private static class MockHMSClientFactory implements HiveConnectionFactory {
+
+ private HiveMetaStoreClient mClient;
+
+ public MockHMSClientFactory() {
+ mClient = null;
+ }
+
+ void setClient(MockClient mockClient) {
+ this.mClient = mockClient.client;
+ }
+ @Override
+ public HMSClient connect() throws IOException, InterruptedException, MetaException {
+ return new HMSClient(mClient);
+ }
+
+ @Override
+ public void close() throws Exception {
+ }
+ }
+
+ /**
+ * Convert Hive snapshot to mock client that will return proper values
+ * for the snapshot.
+ */
+ private static class MockClient {
+
+ public HiveMetaStoreClient client;
+ public long eventId;
+
+ MockClient(HiveSnapshot snapshot, long eventId) throws TException {
+ this.eventId = eventId;
+ client = Mockito.mock(HiveMetaStoreClient.class);
+ List<String> dbNames = new ArrayList<>(snapshot.databases.size());
+ // Walk over all databases and mock appropriate objects
+ for (HiveDb mdb : snapshot.databases) {
+ String dbName = mdb.name;
+ dbNames.add(dbName);
+ Database db = makeDb(dbName);
+ Mockito.when(client.getDatabase(dbName)).thenReturn(db);
+ List<String> tableNames = new ArrayList<>(mdb.tables.size());
+ // Walk over all tables for the database and mock appropriate objects
+ for (HiveTable table : mdb.tables) {
+ String tableName = table.name;
+ tableNames.add(tableName);
+ Table mockTable = makeTable(dbName, tableName);
+ Mockito.when(client.getTableObjectsByName(dbName,
+ Lists.newArrayList(tableName)))
+ .thenReturn(Lists.newArrayList(mockTable));
+ Mockito.when(client.listPartitionNames(dbName, tableName, (short) -1))
+ .thenReturn(table.partitions);
+ // Walk across all partitions and mock appropriate objects
+ for (String partName : table.partitions) {
+ Partition p = makePartition(dbName, tableName, partName);
+ Mockito.when(client.getPartitionsByNames(dbName, tableName,
+ Lists.<String>newArrayList(partName)))
+ .thenReturn(Lists.<Partition>newArrayList(p));
+ }
+ }
+ Mockito.when(client.getAllTables(dbName)).thenReturn(tableNames);
+ }
+ // Return all database names
+ Mockito.when(client.getAllDatabases()).thenReturn(dbNames);
+ Mockito.when(client.getCurrentNotificationEventId()).
+ thenReturn(new CurrentNotificationEventId(eventId));
+
+ }
+
+ public Long incrementNotificationEventId() {
+ eventId = eventId + 1;
+ return eventId;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/sentry/blob/024d99de/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbPrivilegeCleanupOnDrop.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbPrivilegeCleanupOnDrop.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbPrivilegeCleanupOnDrop.java
index b9330cc..d619623 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbPrivilegeCleanupOnDrop.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbPrivilegeCleanupOnDrop.java
@@ -31,7 +31,7 @@ import java.sql.Statement;
import java.util.ArrayList;
import java.util.List;
-import org.apache.sentry.service.thrift.HMSFollower;
+import org.apache.sentry.service.thrift.HmsFollower;
import org.apache.sentry.tests.e2e.hive.AbstractTestWithStaticConfiguration;
import org.junit.BeforeClass;
import org.junit.Before;
@@ -78,7 +78,7 @@ public class TestDbPrivilegeCleanupOnDrop extends
to.close();
// Check the HMS connection only when notification log is enabled.
if (enableNotificationLog) {
- while (!HMSFollower.isConnectedToHMS()) {
+ while (!HmsFollower.isConnectedToHms()) {
Thread.sleep(1000);
}
}