You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sentry.apache.org by ak...@apache.org on 2017/07/28 17:06:51 UTC
[33/50] [abbrv] sentry git commit: SENTRY-1851: Revert "SENTRY-1769
Refactor HMSFollower Class (Kalyan Kumar Kalvagadda reviewed by Vamsee
Yarlagadda, Na Li, Sergio Pena and Alexander Kolbasov)"
http://git-wip-us.apache.org/repos/asf/sentry/blob/07d3ec17/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestHmsFollower.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestHmsFollower.java b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestHmsFollower.java
deleted file mode 100644
index 2095469..0000000
--- a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestHmsFollower.java
+++ /dev/null
@@ -1,532 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sentry.service.thrift;
-
-import static org.mockito.Mockito.reset;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.api.Database;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.NotificationEvent;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.apache.hive.hcatalog.messaging.HCatEventMessage;
-import org.apache.hive.hcatalog.messaging.HCatEventMessage.EventType;
-import org.apache.sentry.binding.metastore.messaging.json.SentryJSONMessageFactory;
-import org.apache.sentry.hdfs.Updateable;
-import org.apache.sentry.provider.db.service.persistent.SentryStore;
-import org.apache.sentry.provider.db.service.thrift.TSentryAuthorizable;
-import org.junit.BeforeClass;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.mockito.Mockito;
-
-import javax.security.auth.login.LoginException;
-
-public class TestHmsFollower {
-
- private final static String hiveInstance = "server2";
- private final static Configuration configuration = new Configuration();
- private final SentryJSONMessageFactory messageFactory = new SentryJSONMessageFactory();
- private final SentryStore sentryStore = Mockito.mock(SentryStore.class);
- private static HiveSimpleConnectionFactory hiveConnectionFactory;
-
- @BeforeClass
- public static void setup() throws IOException, LoginException {
- hiveConnectionFactory = new HiveSimpleConnectionFactory(configuration, new HiveConf());
- hiveConnectionFactory.init();
- configuration.set("sentry.hive.sync.create", "true");
- }
-
- /**
- * Constructs create database event and makes sure that appropriate sentry store API's
- * are invoke when the event is processed by hms follower.
- *
- * @throws Exception
- */
- @Test
- public void testCreateDatabase() throws Exception {
- String dbName = "db1";
-
- // Create notification events
- NotificationEvent notificationEvent = new NotificationEvent(1, 0,
- HCatEventMessage.EventType.CREATE_DATABASE.toString(),
- messageFactory.buildCreateDatabaseMessage(new Database(dbName, null, "hdfs:///db1", null))
- .toString());
- List<NotificationEvent> events = new ArrayList<>();
- events.add(notificationEvent);
- HmsFollower hmsFollower = new HmsFollower(configuration, sentryStore, null,
- hiveConnectionFactory, hiveInstance);
- hmsFollower.processNotifications(events);
-
- TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance);
- authorizable.setServer(hiveInstance);
- authorizable.setDb("db1");
-
- verify(sentryStore, times(1))
- .dropPrivilege(authorizable, NotificationProcessor.getPermUpdatableOnDrop(authorizable));
- }
-
- /**
- * Constructs drop database event and makes sure that appropriate sentry store API's
- * are invoke when the event is processed by hms follower.
- *
- * @throws Exception
- */
- @Test
- public void testDropDatabase() throws Exception {
- String dbName = "db1";
-
- // Create notification events
- NotificationEvent notificationEvent = new NotificationEvent(1, 0,
- HCatEventMessage.EventType.DROP_DATABASE.toString(),
- messageFactory.buildDropDatabaseMessage(new Database(dbName, null, "hdfs:///db1", null))
- .toString());
- List<NotificationEvent> events = new ArrayList<>();
- events.add(notificationEvent);
-
- HmsFollower hmsFollower = new HmsFollower(configuration, sentryStore, null,
- hiveConnectionFactory, hiveInstance);
- hmsFollower.processNotifications(events);
-
- TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance);
- authorizable.setServer(hiveInstance);
- authorizable.setDb("db1");
-
- verify(sentryStore, times(1))
- .dropPrivilege(authorizable, NotificationProcessor.getPermUpdatableOnDrop(authorizable));
- }
-
- /**
- * Constructs create table event and makes sure that appropriate sentry store API's
- * are invoke when the event is processed by hms follower.
- *
- * @throws Exception
- */
- @Test
- public void testCreateTable() throws Exception {
- String dbName = "db1";
- String tableName = "table1";
-
- // Create notification events
- StorageDescriptor sd = new StorageDescriptor();
- sd.setLocation("hdfs:///db1.db/table1");
- NotificationEvent notificationEvent = new NotificationEvent(1, 0,
- HCatEventMessage.EventType.CREATE_TABLE.toString(),
- messageFactory.buildCreateTableMessage(
- new Table(tableName, dbName, null, 0, 0, 0, sd, null, null, null, null, null))
- .toString());
- List<NotificationEvent> events = new ArrayList<>();
- events.add(notificationEvent);
-
- HmsFollower hmsFollower = new HmsFollower(configuration, sentryStore, null,
- hiveConnectionFactory, hiveInstance);
- hmsFollower.processNotifications(events);
-
- TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance);
- authorizable.setServer(hiveInstance);
- authorizable.setDb("db1");
- authorizable.setTable(tableName);
-
- verify(sentryStore, times(1))
- .dropPrivilege(authorizable, NotificationProcessor.getPermUpdatableOnDrop(authorizable));
- }
-
- /**
- * Constructs drop table event and makes sure that appropriate sentry store API's
- * are invoke when the event is processed by hms follower.
- *
- * @throws Exception
- */
- @Test
- public void testDropTable() throws Exception {
- String dbName = "db1";
- String tableName = "table1";
-
- // Create notification events
- StorageDescriptor sd = new StorageDescriptor();
- sd.setLocation("hdfs:///db1.db/table1");
- NotificationEvent notificationEvent = new NotificationEvent(1, 0,
- HCatEventMessage.EventType.DROP_TABLE.toString(),
- messageFactory.buildDropTableMessage(
- new Table(tableName, dbName, null, 0, 0, 0, sd, null, null, null, null, null))
- .toString());
- List<NotificationEvent> events = new ArrayList<>();
- events.add(notificationEvent);
-
- HmsFollower hmsFollower = new HmsFollower(configuration, sentryStore, null,
- hiveConnectionFactory, hiveInstance);
- hmsFollower.processNotifications(events);
-
- TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance);
- authorizable.setServer(hiveInstance);
- authorizable.setDb("db1");
- authorizable.setTable(tableName);
-
- verify(sentryStore, times(1))
- .dropPrivilege(authorizable, NotificationProcessor.getPermUpdatableOnDrop(authorizable));
- }
-
- /**
- * Constructs rename table event and makes sure that appropriate sentry store API's
- * are invoke when the event is processed by hms follower.
- *
- * @throws Exception
- */
- @Test
- public void testRenameTable() throws Exception {
- String dbName = "db1";
- String tableName = "table1";
-
- String newDbName = "db1";
- String newTableName = "table2";
-
- // Create notification events
- StorageDescriptor sd = new StorageDescriptor();
- sd.setLocation("hdfs:///db1.db/table1");
- NotificationEvent notificationEvent = new NotificationEvent(1, 0,
- HCatEventMessage.EventType.ALTER_TABLE.toString(),
- messageFactory.buildAlterTableMessage(
- new Table(tableName, dbName, null, 0, 0, 0, sd, null, null, null, null, null),
- new Table(newTableName, newDbName, null, 0, 0, 0, sd, null, null, null, null, null))
- .toString());
- notificationEvent.setDbName(newDbName);
- notificationEvent.setTableName(newTableName);
- List<NotificationEvent> events = new ArrayList<>();
- events.add(notificationEvent);
-
- HmsFollower hmsFollower = new HmsFollower(configuration, sentryStore, null,
- hiveConnectionFactory, hiveInstance);
- hmsFollower.processNotifications(events);
-
- TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance);
- authorizable.setServer(hiveInstance);
- authorizable.setDb(dbName);
- authorizable.setTable(tableName);
-
- TSentryAuthorizable newAuthorizable = new TSentryAuthorizable(hiveInstance);
- authorizable.setServer(hiveInstance);
- newAuthorizable.setDb(newDbName);
- newAuthorizable.setTable(newTableName);
-
- verify(sentryStore, times(1)).renamePrivilege(authorizable, newAuthorizable,
- NotificationProcessor.getPermUpdatableOnRename(authorizable, newAuthorizable));
- }
-
-
- @Ignore
- /**
- * Constructs a bunch of events and passed to processor of hms follower. One of those is alter
- * partition event with out actually changing anything(invalid event). Idea is to make sure that
- * hms follower calls appropriate sentry store API's for the events processed by hms follower
- * after processing the invalid alter partition event.
- *
- * @throws Exception
- */
- @Test
- public void testAlterPartitionWithInvalidEvent() throws Exception {
- String dbName = "db1";
- String tableName1 = "table1";
- String tableName2 = "table2";
- long inputEventId = 1;
- List<NotificationEvent> events = new ArrayList<>();
- NotificationEvent notificationEvent;
- List<FieldSchema> partCols;
- StorageDescriptor sd;
- Mockito.doNothing().when(sentryStore).persistLastProcessedNotificationID(Mockito.anyLong());
- //noinspection unchecked
- Mockito.doNothing().when(sentryStore).addAuthzPathsMapping(Mockito.anyString(),
- Mockito.anyCollection(), Mockito.any(Updateable.Update.class));
-
- Configuration configuration = new Configuration();
- HmsFollower hmsFollower = new HmsFollower(configuration, sentryStore, null,
- hiveConnectionFactory, hiveInstance);
- // Create a table
- sd = new StorageDescriptor();
- sd.setLocation("hdfs://db1.db/table1");
- partCols = new ArrayList<>();
- partCols.add(new FieldSchema("ds", "string", ""));
- Table table = new Table(tableName1, dbName, null, 0, 0, 0, sd, partCols, null, null, null,
- null);
- notificationEvent = new NotificationEvent(inputEventId, 0,
- HCatEventMessage.EventType.CREATE_TABLE.toString(),
- messageFactory.buildCreateTableMessage(table).toString());
- notificationEvent.setDbName(dbName);
- notificationEvent.setTableName(tableName1);
- events.add(notificationEvent);
- inputEventId += 1;
- // Process the notification
- hmsFollower.processNotifications(events);
- // Make sure that addAuthzPathsMapping was invoked once to handle CREATE_TABLE notification
- // and persistLastProcessedNotificationID was not invoked.
- //noinspection unchecked
- verify(sentryStore, times(1)).addAuthzPathsMapping(Mockito.anyString(),
- Mockito.anyCollection(), Mockito.any(Updateable.Update.class));
- verify(sentryStore, times(0)).persistLastProcessedNotificationID(Mockito.anyLong());
- reset(sentryStore);
- events.clear();
-
- // Create a partition
- List<Partition> partitions = new ArrayList<>();
- StorageDescriptor invalidSd = new StorageDescriptor();
- invalidSd.setLocation(null);
- Partition partition = new Partition(Collections.singletonList("today"), dbName, tableName1,
- 0, 0, sd, null);
- partitions.add(partition);
- notificationEvent = new NotificationEvent(inputEventId, 0, EventType.ADD_PARTITION.toString(),
- messageFactory.buildAddPartitionMessage(table, partitions).toString());
- notificationEvent.setDbName(dbName);
- notificationEvent.setTableName(tableName1);
- events.add(notificationEvent);
- inputEventId += 1;
- //Process the notification
- hmsFollower.processNotifications(events);
- // Make sure that addAuthzPathsMapping was invoked once to handle ADD_PARTITION notification
- // and persistLastProcessedNotificationID was not invoked.
- //noinspection unchecked
- verify(sentryStore, times(1)).addAuthzPathsMapping(Mockito.anyString(),
- Mockito.anyCollection(), Mockito.any(Updateable.Update.class));
- verify(sentryStore, times(0)).persistLastProcessedNotificationID(Mockito.anyLong());
- reset(sentryStore);
- events.clear();
-
- // Create a alter notification with out actually changing anything.
- // This is an invalid event and should be processed by sentry store.
- // Event Id should be explicitly persisted using persistLastProcessedNotificationID
- notificationEvent = new NotificationEvent(inputEventId, 0, EventType.ALTER_PARTITION.toString(),
- messageFactory.buildAlterPartitionMessage(partition, partition).toString());
- notificationEvent.setDbName(dbName);
- notificationEvent.setTableName(tableName1);
- events.add(notificationEvent);
- inputEventId += 1;
- // Process the notification
- hmsFollower.processNotifications(events);
- // Make sure that persistLastProcessedNotificationID is invoked explicitly.
- verify(sentryStore, times(1)).persistLastProcessedNotificationID(inputEventId - 1);
- reset(sentryStore);
- events.clear();
-
- // Create a alter notification with some actual change.
- sd = new StorageDescriptor();
- sd.setLocation("hdfs://user/hive/warehouse/db1.db/table1");
- Partition updatedPartition = new Partition(partition);
- updatedPartition.setSd(sd);
- notificationEvent = new NotificationEvent(inputEventId, 0, EventType.ALTER_PARTITION.toString(),
- messageFactory.buildAlterPartitionMessage(partition, updatedPartition).toString());
- notificationEvent.setDbName(dbName);
- notificationEvent.setTableName(tableName1);
- events.add(notificationEvent);
- inputEventId += 1;
- // Process the notification
- hmsFollower.processNotifications(events);
- // Make sure that updateAuthzPathsMapping was invoked once to handle ALTER_PARTITION
- // notification and persistLastProcessedNotificationID was not invoked.
- verify(sentryStore, times(1)).updateAuthzPathsMapping(Mockito.anyString(),
- Mockito.anyString(), Mockito.anyString(), Mockito.any(Updateable.Update.class));
- verify(sentryStore, times(0)).persistLastProcessedNotificationID(inputEventId - 1);
- reset(sentryStore);
- events.clear();
-
- // Create a table
- sd = new StorageDescriptor();
- sd.setLocation("hdfs://db1.db/table2");
- partCols = new ArrayList<>();
- partCols.add(new FieldSchema("ds", "string", ""));
- Table table1 = new Table(tableName2, dbName, null, 0, 0, 0, sd, partCols, null, null, null,
- null);
- notificationEvent = new NotificationEvent(inputEventId, 0,
- HCatEventMessage.EventType.CREATE_TABLE.toString(),
- messageFactory.buildCreateTableMessage(table1).toString());
- notificationEvent.setDbName(dbName);
- notificationEvent.setTableName(tableName2);
- events.add(notificationEvent);
- // Process the notification
- hmsFollower.processNotifications(events);
- // Make sure that addAuthzPathsMapping was invoked once to handle CREATE_TABLE notification
- // and persistLastProcessedNotificationID was not invoked.
- //noinspection unchecked
- verify(sentryStore, times(1)).addAuthzPathsMapping(Mockito.anyString(),
- Mockito.anyCollection(), Mockito.any(Updateable.Update.class));
- verify(sentryStore, times(0)).persistLastProcessedNotificationID(Mockito.anyLong());
- }
-
- /**
- * Constructs a bunch of events and passed to processor of hms follower. One of those is alter
- * table event with out actually changing anything(invalid event). Idea is to make sure that
- * hms follower calls appropriate sentry store API's for the events processed by hms follower
- * after processing the invalid alter table event.
- *
- * @throws Exception
- */
- @Test
- public void testAlterTableWithInvalidEvent() throws Exception {
- String dbName = "db1";
- String tableName1 = "table1";
- String tableName2 = "table2";
- long inputEventId = 1;
- List<NotificationEvent> events = new ArrayList<>();
- NotificationEvent notificationEvent;
- List<FieldSchema> partCols;
- StorageDescriptor sd;
- Mockito.doNothing().when(sentryStore).persistLastProcessedNotificationID(Mockito.anyLong());
- //noinspection unchecked
- Mockito.doNothing().when(sentryStore).addAuthzPathsMapping(Mockito.anyString(),
- Mockito.anyCollection(), Mockito.any(Updateable.Update.class));
-
- Configuration configuration = new Configuration();
- HmsFollower hmsFollower = new HmsFollower(configuration, sentryStore, null,
- hiveConnectionFactory, hiveInstance);
-
- // Create a table
- sd = new StorageDescriptor();
- sd.setLocation("hdfs://db1.db/table1");
- partCols = new ArrayList<>();
- partCols.add(new FieldSchema("ds", "string", ""));
- Table table = new Table(tableName1, dbName, null, 0, 0, 0, sd, partCols, null, null, null,
- null);
- notificationEvent = new NotificationEvent(inputEventId, 0,
- HCatEventMessage.EventType.CREATE_TABLE.toString(),
- messageFactory.buildCreateTableMessage(table).toString());
- notificationEvent.setDbName(dbName);
- notificationEvent.setTableName(tableName1);
- events.add(notificationEvent);
- inputEventId += 1;
- // Process the notification
- hmsFollower.processNotifications(events);
- // Make sure that addAuthzPathsMapping was invoked once to handle CREATE_TABLE notification
- // and persistLastProcessedNotificationID was not invoked.
- //noinspection unchecked
- verify(sentryStore, times(1)).addAuthzPathsMapping(Mockito.anyString(),
- Mockito.anyCollection(), Mockito.any(Updateable.Update.class));
- verify(sentryStore, times(0)).persistLastProcessedNotificationID(Mockito.anyLong());
- reset(sentryStore);
- events.clear();
-
- // Create alter table notification with out actually changing anything.
- // This notification should not be processed by sentry server
- // Notification should be persisted explicitly
- notificationEvent = new NotificationEvent(1, 0,
- HCatEventMessage.EventType.ALTER_TABLE.toString(),
- messageFactory.buildAlterTableMessage(
- new Table(tableName1, dbName, null, 0, 0, 0, sd, null, null, null, null, null),
- new Table(tableName1, dbName, null, 0, 0, 0, sd, null, null, null, null, null))
- .toString());
- notificationEvent.setDbName(dbName);
- notificationEvent.setTableName(tableName1);
- events = new ArrayList<>();
- events.add(notificationEvent);
- inputEventId += 1;
- // Process the notification
- hmsFollower.processNotifications(events);
- // Make sure that renameAuthzObj and deleteAuthzPathsMapping were not invoked
- // to handle CREATE_TABLE notification
- // and persistLastProcessedNotificationID is explicitly invoked
- verify(sentryStore, times(0)).renameAuthzObj(Mockito.anyString(), Mockito.anyString(),
- Mockito.any(Updateable.Update.class));
- //noinspection unchecked
- verify(sentryStore, times(0)).deleteAuthzPathsMapping(Mockito.anyString(),
- Mockito.anyCollection(), Mockito.any(Updateable.Update.class));
- verify(sentryStore, times(1)).persistLastProcessedNotificationID(Mockito.anyLong());
- reset(sentryStore);
- events.clear();
-
- // Create a table
- sd = new StorageDescriptor();
- sd.setLocation("hdfs://db1.db/table2");
- partCols = new ArrayList<>();
- partCols.add(new FieldSchema("ds", "string", ""));
- Table table1 = new Table(tableName2, dbName, null, 0, 0, 0, sd, partCols, null, null, null,
- null);
- notificationEvent = new NotificationEvent(inputEventId, 0,
- HCatEventMessage.EventType.CREATE_TABLE.toString(),
- messageFactory.buildCreateTableMessage(table1).toString());
- notificationEvent.setDbName(dbName);
- notificationEvent.setTableName(tableName2);
- events.add(notificationEvent);
- // Process the notification
- hmsFollower.processNotifications(events);
- // Make sure that addAuthzPathsMapping was invoked once to handle CREATE_TABLE notification
- // and persistLastProcessedNotificationID was not invoked.
- //noinspection unchecked
- verify(sentryStore, times(1)).addAuthzPathsMapping(Mockito.anyString(),
- Mockito.anyCollection(), Mockito.any(Updateable.Update.class));
- verify(sentryStore, times(0)).persistLastProcessedNotificationID(Mockito.anyLong());
- }
-
- /**
- * Constructs a two events and passed to processor of hms follower. First one being create table
- * event with location information(Invalid Event). Idea is to make sure that hms follower calls
- * appropriate sentry store API's for the event processed by hms follower after processing the
- * invalid create table event.
- *
- * @throws Exception
- */
- public void testCreateTableAfterInvalidEvent() throws Exception {
- String dbName = "db1";
- String tableName = "table1";
- long inputEventId = 1;
-
- Mockito.doNothing().when(sentryStore).persistLastProcessedNotificationID(Mockito.anyLong());
- //noinspection unchecked
- Mockito.doNothing().when(sentryStore)
- .addAuthzPathsMapping(Mockito.anyString(), Mockito.anyCollection(),
- Mockito.any(Updateable.Update.class));
-
- // Create invalid notification event. The location of the storage descriptor is null, which is invalid for creating table
- StorageDescriptor invalidSd = new StorageDescriptor();
- invalidSd.setLocation(null);
- NotificationEvent invalidNotificationEvent = new NotificationEvent(inputEventId, 0,
- HCatEventMessage.EventType.CREATE_TABLE.toString(),
- messageFactory.buildCreateTableMessage(
- new Table(tableName, dbName, null, 0, 0, 0, invalidSd, null, null, null, null, null))
- .toString());
-
- // Create valid notification event
- StorageDescriptor sd = new StorageDescriptor();
- sd.setLocation("hdfs://db1.db/table1");
- inputEventId += 1;
- NotificationEvent notificationEvent = new NotificationEvent(inputEventId, 0,
- HCatEventMessage.EventType.CREATE_TABLE.toString(),
- messageFactory.buildCreateTableMessage(
- new Table(tableName, dbName, null, 0, 0, 0, sd, null, null, null, null, null))
- .toString());
- List<NotificationEvent> events = new ArrayList<>();
- events.add(invalidNotificationEvent);
- events.add(notificationEvent);
-
- Configuration configuration = new Configuration();
- HmsFollower hmsFollower = new HmsFollower(configuration, sentryStore, null,
- hiveConnectionFactory, hiveInstance);
- hmsFollower.processNotifications(events);
-
- // invalid event updates notification ID directly
- verify(sentryStore, times(1)).persistLastProcessedNotificationID(inputEventId - 1);
-
- // next valid event update path, which updates notification ID
- //noinspection unchecked
- verify(sentryStore, times(1)).addAuthzPathsMapping(Mockito.anyString(), Mockito.anyCollection(),
- Mockito.any(Updateable.Update.class));
- }
-}
http://git-wip-us.apache.org/repos/asf/sentry/blob/07d3ec17/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestNotificationProcessor.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestNotificationProcessor.java b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestNotificationProcessor.java
deleted file mode 100644
index c6c9448..0000000
--- a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestNotificationProcessor.java
+++ /dev/null
@@ -1,465 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sentry.service.thrift;
-
-import static org.mockito.Mockito.reset;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-
-import java.util.ArrayList;
-import java.util.List;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.metastore.api.Database;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.NotificationEvent;
-import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.apache.hive.hcatalog.messaging.HCatEventMessage;
-import org.apache.sentry.binding.metastore.messaging.json.SentryJSONMessageFactory;
-import org.apache.sentry.hdfs.Updateable;
-import org.apache.sentry.provider.db.service.persistent.SentryStore;
-import org.apache.sentry.provider.db.service.thrift.TSentryAuthorizable;
-import org.junit.After;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.mockito.Mockito;
-
-// TODO 1. More tests should be added here.
-// TODO 2. Tests using actual sentry store where.
-@SuppressWarnings("unused")
-public class TestNotificationProcessor {
-
- private static final SentryStore sentryStore = Mockito.mock(SentryStore.class);
- private final static String hiveInstance = "server2";
- private final static Configuration conf = new Configuration();
- private final SentryJSONMessageFactory messageFactory = new SentryJSONMessageFactory();
- private NotificationProcessor notificationProcessor;
-
- @BeforeClass
- public static void setup() {
- conf.set("sentry.hive.sync.create", "true");
- conf.set("sentry.hive.sync.drop", "true");
- }
-
- @After
- public void resetConf() {
- conf.set("sentry.hive.sync.create", "true");
- conf.set("sentry.hive.sync.drop", "true");
- reset(sentryStore);
- }
-
- @Test
- /*
- Makes sure that appropriate sentry store methods are invoked when create database event is
- processed.
-
- Also, checks the hive sync configuration.
- */
- public void testCreateDatabase() throws Exception {
- long seqNum = 1;
- String dbName = "db1";
- String uriPrefix = "hdfs:///";
- String location = "user/hive/warehouse";
- NotificationEvent notificationEvent;
- TSentryAuthorizable authorizable;
- notificationProcessor = new NotificationProcessor(sentryStore,
- hiveInstance, conf);
-
- // Create notification event
- notificationEvent = new NotificationEvent(seqNum, 0,
- HCatEventMessage.EventType.CREATE_DATABASE.toString(),
- messageFactory.buildCreateDatabaseMessage(new Database(dbName,
- null, uriPrefix + location, null)).toString());
-
- notificationProcessor.processNotificationEvent(notificationEvent);
-
- authorizable = new TSentryAuthorizable(hiveInstance);
- authorizable.setServer(hiveInstance);
- authorizable.setDb("db1");
- //noinspection unchecked
- verify(sentryStore, times(1)).addAuthzPathsMapping(Mockito.anyString(),
- Mockito.anyCollection(), Mockito.any(Updateable.Update.class));
-
- verify(sentryStore, times(1)).dropPrivilege(authorizable,
- NotificationProcessor.getPermUpdatableOnDrop(authorizable));
- reset(sentryStore);
-
- //Change the configuration and make sure that exiting privileges are not dropped
- notificationProcessor.setSyncStoreOnCreate(false);
- dbName = "db2";
- notificationEvent = new NotificationEvent(1, 0,
- HCatEventMessage.EventType.CREATE_DATABASE.toString(),
- messageFactory.buildCreateDatabaseMessage(new Database(dbName,
- null, "hdfs:///db2", null)).toString());
-
- notificationProcessor.processNotificationEvent(notificationEvent);
-
- authorizable = new TSentryAuthorizable(hiveInstance);
- authorizable.setServer(hiveInstance);
- authorizable.setDb(dbName);
-
- //noinspection unchecked
- verify(sentryStore, times(1)).addAuthzPathsMapping(Mockito.anyString(),
- Mockito.anyCollection(), Mockito.any(Updateable.Update.class));
- //making sure that privileges are not dropped
- verify(sentryStore, times(0)).dropPrivilege(authorizable,
- NotificationProcessor.getPermUpdatableOnDrop(authorizable));
-
- }
-
- @Test
- /*
- Makes sure that appropriate sentry store methods are invoked when drop database event is
- processed.
-
- Also, checks the hive sync configuration.
- */
- public void testDropDatabase() throws Exception {
- String dbName = "db1";
-
- notificationProcessor = new NotificationProcessor(sentryStore,
- hiveInstance, conf);
-
- // Create notification event
- NotificationEvent notificationEvent = new NotificationEvent(1, 0,
- HCatEventMessage.EventType.DROP_DATABASE.toString(),
- messageFactory.buildDropDatabaseMessage(new Database(dbName, null,
- "hdfs:///db1", null)).toString());
-
- notificationProcessor.processNotificationEvent(notificationEvent);
-
- TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance);
- authorizable.setServer(hiveInstance);
- authorizable.setDb("db1");
-
- //noinspection unchecked
- verify(sentryStore, times(1)).deleteAuthzPathsMapping(Mockito.anyString(),
- Mockito.anyCollection(), Mockito.any(Updateable.Update.class));
- verify(sentryStore, times(1)).dropPrivilege(authorizable,
- NotificationProcessor.getPermUpdatableOnDrop(authorizable));
- reset(sentryStore);
-
- // Change the configuration and make sure that exiting privileges are not dropped
- notificationProcessor.setSyncStoreOnDrop(false);
- dbName = "db2";
- // Create notification event
- notificationEvent = new NotificationEvent(1, 0,
- HCatEventMessage.EventType.DROP_DATABASE.toString(),
- messageFactory.buildDropDatabaseMessage(new Database(dbName, null,
- "hdfs:///db2", null)).toString());
-
- notificationProcessor.processNotificationEvent(notificationEvent);
-
- authorizable = new TSentryAuthorizable(hiveInstance);
- authorizable.setServer(hiveInstance);
- authorizable.setDb(dbName);
-
- //noinspection unchecked
- verify(sentryStore, times(1)).deleteAuthzPathsMapping(Mockito.anyString(),
- Mockito.anyCollection(), Mockito.any(Updateable.Update.class));
- verify(sentryStore, times(0)).dropPrivilege(authorizable,
- NotificationProcessor.getPermUpdatableOnDrop(authorizable));
- }
-
- @Test
- /*
- Makes sure that appropriate sentry store methods are invoked when create table event is
- processed.
-
- Also, checks the hive sync configuration.
- */
- public void testCreateTable() throws Exception {
- String dbName = "db1";
- String tableName = "table1";
-
- notificationProcessor = new NotificationProcessor(sentryStore,
- hiveInstance, conf);
-
- // Create notification event
- StorageDescriptor sd = new StorageDescriptor();
- sd.setLocation("hdfs:///db1.db/table1");
- NotificationEvent notificationEvent =
- new NotificationEvent(1, 0, HCatEventMessage.EventType.CREATE_TABLE.toString(),
- messageFactory.buildCreateTableMessage(new Table(tableName,
- dbName, null, 0, 0, 0, sd, null, null, null, null, null)).toString());
-
- notificationProcessor.processNotificationEvent(notificationEvent);
-
- TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance);
- authorizable.setServer(hiveInstance);
- authorizable.setDb("db1");
- authorizable.setTable(tableName);
-
- //noinspection unchecked
- verify(sentryStore, times(1)).addAuthzPathsMapping(Mockito.anyString(),
- Mockito.anyCollection(), Mockito.any(Updateable.Update.class));
-
- verify(sentryStore, times(1)).dropPrivilege(authorizable,
- NotificationProcessor.getPermUpdatableOnDrop(authorizable));
- reset(sentryStore);
-
- // Change the configuration and make sure that existing privileges are not dropped
- notificationProcessor.setSyncStoreOnCreate(false);
-
- // Create notification event
- dbName = "db2";
- tableName = "table2";
- sd = new StorageDescriptor();
- sd.setLocation("hdfs:///db1.db/table2");
- notificationEvent =
- new NotificationEvent(1, 0, HCatEventMessage.EventType.CREATE_TABLE.toString(),
- messageFactory.buildCreateTableMessage(new Table(tableName,
- dbName, null, 0, 0, 0, sd, null, null, null, null, null)).toString());
-
- notificationProcessor.processNotificationEvent(notificationEvent);
-
- authorizable = new TSentryAuthorizable(hiveInstance);
- authorizable.setServer(hiveInstance);
- authorizable.setDb(dbName);
- authorizable.setTable(tableName);
-
- //noinspection unchecked
- verify(sentryStore, times(1)).addAuthzPathsMapping(Mockito.anyString(),
- Mockito.anyCollection(), Mockito.any(Updateable.Update.class));
- // Making sure that privileges are not dropped
- verify(sentryStore, times(0)).dropPrivilege(authorizable,
- NotificationProcessor.getPermUpdatableOnDrop(authorizable));
- }
-
- @Test
- /*
- Makes sure that appropriate sentry store methods are invoked when drop table event is
- processed.
-
- Also, checks the hive sync configuration.
- */
- public void testDropTable() throws Exception {
- String dbName = "db1";
- String tableName = "table1";
-
- Configuration authConf = new Configuration();
- notificationProcessor = new NotificationProcessor(sentryStore,
- hiveInstance, authConf);
-
- // Create notification event
- StorageDescriptor sd = new StorageDescriptor();
- sd.setLocation("hdfs:///db1.db/table1");
- NotificationEvent notificationEvent = new NotificationEvent(1, 0,
- HCatEventMessage.EventType.DROP_TABLE.toString(),
- messageFactory.buildDropTableMessage(new Table(tableName,
- dbName, null, 0, 0, 0, sd, null, null, null, null, null)).toString());
-
- notificationProcessor.processNotificationEvent(notificationEvent);
-
- TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance);
- authorizable.setServer(hiveInstance);
- authorizable.setDb("db1");
- authorizable.setTable(tableName);
-
- verify(sentryStore, times(1)).deleteAllAuthzPathsMapping(Mockito.anyString(),
- Mockito.any(Updateable.Update.class));
-
- verify(sentryStore, times(1)).dropPrivilege(authorizable,
- NotificationProcessor.getPermUpdatableOnDrop(authorizable));
- }
-
- @Test
- /*
- Makes sure that appropriate sentry store methods are invoked when alter tables event is
- processed.
- */
- public void testAlterTable() throws Exception {
- String dbName = "db1";
- String tableName = "table1";
-
- String newDbName = "db1";
- String newTableName = "table2";
-
- Configuration authConf = new Configuration();
- notificationProcessor = new NotificationProcessor(sentryStore,
- hiveInstance, authConf);
-
- // Create notification event
- StorageDescriptor sd = new StorageDescriptor();
- sd.setLocation("hdfs:///db1.db/table1");
- NotificationEvent notificationEvent = new NotificationEvent(1, 0,
- HCatEventMessage.EventType.ALTER_TABLE.toString(),
- messageFactory.buildAlterTableMessage(
- new Table(tableName, dbName, null, 0, 0, 0, sd, null, null, null, null, null),
- new Table(newTableName, newDbName, null, 0, 0, 0, sd, null, null, null, null, null))
- .toString());
- notificationEvent.setDbName(newDbName);
- notificationEvent.setTableName(newTableName);
-
- notificationProcessor.processNotificationEvent(notificationEvent);
-
- TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance);
- authorizable.setServer(hiveInstance);
- authorizable.setDb(dbName);
- authorizable.setTable(tableName);
-
- TSentryAuthorizable newAuthorizable = new TSentryAuthorizable(hiveInstance);
- authorizable.setServer(hiveInstance);
- newAuthorizable.setDb(newDbName);
- newAuthorizable.setTable(newTableName);
-
- verify(sentryStore, times(1)).renameAuthzObj(Mockito.anyString(), Mockito.anyString(),
- Mockito.any(Updateable.Update.class));
-
- verify(sentryStore, times(1)).renamePrivilege(authorizable, newAuthorizable,
- NotificationProcessor.getPermUpdatableOnRename(authorizable, newAuthorizable));
- }
-
- @Test
- /*
- Makes sure that appropriate sentry store methods are invoked when alter tables event is
- processed.
- */
- public void testRenameTableWithLocationUpdate() throws Exception {
- String dbName = "db1";
- String tableName = "table1";
-
- String newDbName = "db1";
- String newTableName = "table2";
-
- Configuration authConf = new Configuration();
- notificationProcessor = new NotificationProcessor(sentryStore,
- hiveInstance, authConf);
-
- // Create notification event
- StorageDescriptor sd = new StorageDescriptor();
- sd.setLocation("hdfs:///db1.db/table1");
- StorageDescriptor new_sd = new StorageDescriptor();
- new_sd.setLocation("hdfs:///db1.db/table2");
- NotificationEvent notificationEvent = new NotificationEvent(1, 0,
- HCatEventMessage.EventType.ALTER_TABLE.toString(),
- messageFactory.buildAlterTableMessage(
- new Table(tableName, dbName, null, 0, 0, 0, sd, null, null, null, null, null),
- new Table(newTableName, newDbName, null, 0, 0, 0, new_sd, null, null, null, null, null))
- .toString());
- notificationEvent.setDbName(newDbName);
- notificationEvent.setTableName(newTableName);
-
- notificationProcessor.processNotificationEvent(notificationEvent);
-
- TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance);
- authorizable.setServer(hiveInstance);
- authorizable.setDb(dbName);
- authorizable.setTable(tableName);
-
- TSentryAuthorizable newAuthorizable = new TSentryAuthorizable(hiveInstance);
- authorizable.setServer(hiveInstance);
- newAuthorizable.setDb(newDbName);
- newAuthorizable.setTable(newTableName);
-
- verify(sentryStore, times(1)).renameAuthzPathsMapping(Mockito.anyString(), Mockito.anyString(),
- Mockito.anyString(), Mockito.anyString(), Mockito.any(Updateable.Update.class));
-
- verify(sentryStore, times(1)).renamePrivilege(authorizable, newAuthorizable,
- NotificationProcessor.getPermUpdatableOnRename(authorizable, newAuthorizable));
- }
-
- @Test
- /*
- Test to made sure that sentry store is not invoked when invalid alter table event is
- processed.
- */
- public void testAlterTableWithInvalidEvent() throws Exception {
- String dbName = "db1";
- String tableName1 = "table1";
- String tableName2 = "table2";
- long inputEventId = 1;
- NotificationEvent notificationEvent;
- List<FieldSchema> partCols;
- StorageDescriptor sd;
- Mockito.doNothing().when(sentryStore).persistLastProcessedNotificationID(Mockito.anyLong());
- //noinspection unchecked
- Mockito.doNothing().when(sentryStore).addAuthzPathsMapping(Mockito.anyString(),
- Mockito.anyCollection(), Mockito.any(Updateable.Update.class));
-
- Configuration authConf = new Configuration();
- notificationProcessor = new NotificationProcessor(sentryStore,
- hiveInstance, authConf);
-
- // Create a table
- sd = new StorageDescriptor();
- sd.setLocation("hdfs://db1.db/table1");
- partCols = new ArrayList<>();
- partCols.add(new FieldSchema("ds", "string", ""));
- Table table = new Table(tableName1, dbName, null, 0, 0, 0, sd, partCols,
- null, null, null, null);
- notificationEvent = new NotificationEvent(inputEventId, 0,
- HCatEventMessage.EventType.CREATE_TABLE.toString(),
- messageFactory.buildCreateTableMessage(table).toString());
- notificationEvent.setDbName(dbName);
- notificationEvent.setTableName(tableName1);
- inputEventId += 1;
- // Process the notification
- notificationProcessor.processNotificationEvent(notificationEvent);
- // Make sure that addAuthzPathsMapping was invoked once to handle CREATE_TABLE notification
- // and persistLastProcessedNotificationID was not invoked.
- //noinspection unchecked
- verify(sentryStore, times(1)).addAuthzPathsMapping(Mockito.anyString(),
- Mockito.anyCollection(), Mockito.any(Updateable.Update.class));
- reset(sentryStore);
-
- // Create alter table notification with out actually changing anything.
- // This notification should not be processed by sentry server
- // Notification should be persisted explicitly
- notificationEvent = new NotificationEvent(1, 0,
- HCatEventMessage.EventType.ALTER_TABLE.toString(),
- messageFactory.buildAlterTableMessage(
- new Table(tableName1, dbName, null, 0, 0, 0, sd, null, null, null, null, null),
- new Table(tableName1, dbName, null, 0, 0, 0, sd, null,
- null, null, null, null)).toString());
- notificationEvent.setDbName(dbName);
- notificationEvent.setTableName(tableName1);
- inputEventId += 1;
- // Process the notification
- notificationProcessor.processNotificationEvent(notificationEvent);
- // Make sure that renameAuthzObj and deleteAuthzPathsMapping were not invoked
- // to handle CREATE_TABLE notification
- // and persistLastProcessedNotificationID is explicitly invoked
- verify(sentryStore, times(0)).renameAuthzObj(Mockito.anyString(), Mockito.anyString(),
- Mockito.any(Updateable.Update.class));
- //noinspection unchecked
- verify(sentryStore, times(0)).deleteAuthzPathsMapping(Mockito.anyString(),
- Mockito.anyCollection(), Mockito.any(Updateable.Update.class));
- reset(sentryStore);
-
- // Create a table
- sd = new StorageDescriptor();
- sd.setLocation("hdfs://db1.db/table2");
- partCols = new ArrayList<>();
- partCols.add(new FieldSchema("ds", "string", ""));
- Table table1 = new Table(tableName2, dbName, null, 0, 0, 0, sd,
- partCols, null, null, null, null);
- notificationEvent = new NotificationEvent(inputEventId, 0,
- HCatEventMessage.EventType.CREATE_TABLE.toString(),
- messageFactory.buildCreateTableMessage(table1).toString());
- notificationEvent.setDbName(dbName);
- notificationEvent.setTableName(tableName2);
- // Process the notification
- notificationProcessor.processNotificationEvent(notificationEvent);
- // Make sure that addAuthzPathsMapping was invoked once to handle CREATE_TABLE notification
- // and persistLastProcessedNotificationID was not invoked.
- //noinspection unchecked
- verify(sentryStore, times(1)).addAuthzPathsMapping(Mockito.anyString(),
- Mockito.anyCollection(), Mockito.any(Updateable.Update.class));
- }
-}
http://git-wip-us.apache.org/repos/asf/sentry/blob/07d3ec17/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestSentryHmsClient.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestSentryHmsClient.java b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestSentryHmsClient.java
deleted file mode 100644
index 3cc6541..0000000
--- a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestSentryHmsClient.java
+++ /dev/null
@@ -1,470 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.sentry.service.thrift;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.Random;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
-import org.apache.hadoop.hive.metastore.IMetaStoreClient.NotificationFilter;
-import org.apache.hadoop.hive.metastore.api.CurrentNotificationEventId;
-import org.apache.hadoop.hive.metastore.api.Database;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.NotificationEvent;
-import org.apache.hadoop.hive.metastore.api.NotificationEventResponse;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.apache.hive.hcatalog.messaging.HCatEventMessage;
-import org.apache.sentry.binding.metastore.messaging.json.SentryJSONMessageFactory;
-import org.apache.sentry.provider.db.service.persistent.PathsImage;
-import org.apache.thrift.TException;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.mockito.Mockito;
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
-
-import javax.security.auth.login.LoginException;
-
-/**
- * Test mocks HiveMetaStoreClient class and tests SentryHmsClient.
- */
-public class TestSentryHmsClient {
-
- private static final Configuration conf = new Configuration();
- private static final SentryJSONMessageFactory messageFactory = new SentryJSONMessageFactory();
- private static SentryHmsClient client;
- private static MockHMSClientFactory hiveConnectionFactory;
-
- /**
- * Create mock database with the given name
- *
- * @param name Database name
- * @return Mock database object
- */
- private static Database makeDb(String name) {
- Database db = Mockito.mock(Database.class);
- Mockito.when(db.getName()).thenReturn(name);
- Mockito.when(db.getLocationUri()).thenReturn("hdfs:///" + name);
- return db;
- }
-
- /**
- * Create mock table
- *
- * @param dbName db for this table
- * @param tableName name of the table
- * @return mock table object
- */
- private static Table makeTable(String dbName, String tableName) {
- Table table = Mockito.mock(Table.class);
- Mockito.when(table.getDbName()).thenReturn(dbName);
- Mockito.when(table.getTableName()).thenReturn(tableName);
- StorageDescriptor sd = Mockito.mock(StorageDescriptor.class);
- Mockito.when(sd.getLocation()).thenReturn(
- String.format("hdfs:///%s/%s", dbName, tableName));
- Mockito.when(table.getSd()).thenReturn(sd);
- return table;
- }
-
- /**
- * Create mock partition
- *
- * @param dbName database for this partition
- * @param tableName table for this partition
- * @param partName partition name
- * @return mock partition object
- */
- private static Partition makePartition(String dbName, String tableName, String partName) {
- Partition partition = Mockito.mock(Partition.class);
- StorageDescriptor sd = Mockito.mock(StorageDescriptor.class);
- Mockito.when(sd.getLocation()).thenReturn(
- String.format("hdfs:///%s/%s/%s", dbName, tableName, partName));
- Mockito.when(partition.getSd()).thenReturn(sd);
- return partition;
- }
-
- /**
- * Creates create database notification
- *
- * @return NotificationEvent
- */
- private static NotificationEvent getCreateDatabaseNotification(long id) {
- Random rand = new Random();
- int n = rand.nextInt(100) + 1;
- String dbName = "db" + n;
- return new NotificationEvent(id, 0, HCatEventMessage.EventType.CREATE_DATABASE.toString(),
- messageFactory
- .buildCreateDatabaseMessage(new Database(dbName, null, "hdfs:///" + dbName, null))
- .toString());
- }
-
- /**
- * Creates drop database notification
- *
- * @return NotificationEvent
- */
- private static NotificationEvent getDropDatabaseNotification(long id) {
- Random rand = new Random();
- int n = rand.nextInt(100) + 1;
- String dbName = "db" + n;
- return new NotificationEvent(id, 0, HCatEventMessage.EventType.DROP_DATABASE.toString(),
- messageFactory
- .buildDropDatabaseMessage(new Database(dbName, null, "hdfs:///" + dbName, null))
- .toString());
- }
-
- @BeforeClass
- static public void initialize() throws IOException, LoginException {
- hiveConnectionFactory = new MockHMSClientFactory();
- client = new SentryHmsClient(conf, (HiveConnectionFactory)hiveConnectionFactory);
- }
-
- /**
- * Creating snapshot when SentryHmsClient is not connected to HMS
- */
- @Test
- public void testSnapshotCreationWithOutClientConnected() throws Exception {
- // Make sure that client is not connected
- Assert.assertFalse(client.isConnected());
- PathsImage snapshotInfo = client.getFullSnapshot();
- Assert.assertTrue(snapshotInfo.getPathImage().isEmpty());
- }
-
- /**
- * Creating snapshot when HMS doesn't have any data
- */
- @Test
- public void testSnapshotCreationWithNoHmsData() throws Exception {
- MockClient mockClient = new MockClient(new HiveSnapshot(), 1);
- client.setClient(mockClient.client);
- // Make sure that client is connected
- Assert.assertTrue(client.isConnected());
- PathsImage snapshotInfo = client.getFullSnapshot();
- Assert.assertTrue(snapshotInfo.getPathImage().isEmpty());
- }
-
- /**
- * Creating a snapshot when there is data but there are updates to HMS data mean while
- */
- @Test
- public void testSnapshotCreationWhenDataIsActivelyUpdated() throws Exception {
- HiveTable tab21 = new HiveTable("tab21");
- HiveTable tab31 = new HiveTable("tab31").add("part311").add("part312");
- HiveDb db3 = new HiveDb("db3", Lists.newArrayList(tab31));
- HiveDb db2 = new HiveDb("db2", Lists.newArrayList(tab21));
- HiveDb db1 = new HiveDb("db1");
- HiveSnapshot snap = new HiveSnapshot().add(db1).add(db2).add(db3);
- final MockClient mockClient = new MockClient(snap, 1);
-
- client.setClient(mockClient.client);
- hiveConnectionFactory.setClient(mockClient);
- // Make sure that client is connected
- Assert.assertTrue(client.isConnected());
- PathsImage snapshotInfo = client.getFullSnapshot();
- // Make sure that snapshot is not empty
- Assert.assertTrue(!snapshotInfo.getPathImage().isEmpty());
-
- Mockito.when(mockClient.client.getCurrentNotificationEventId()).
- thenAnswer(new Answer<CurrentNotificationEventId>() {
- @Override
- public CurrentNotificationEventId answer(InvocationOnMock invocation)
- throws Throwable {
- return new CurrentNotificationEventId(mockClient.incrementNotificationEventId());
- }
-
- });
-
- snapshotInfo = client.getFullSnapshot();
- Assert.assertTrue(snapshotInfo.getPathImage().isEmpty());
- }
-
- /**
- * Creating a snapshot when there is data in HMS.
- */
- @Test
- public void testSnapshotCreationSuccess() throws Exception {
- HiveTable tab21 = new HiveTable("tab21");
- HiveTable tab31 = new HiveTable("tab31");
- HiveDb db3 = new HiveDb("db3", Lists.newArrayList(tab31));
- HiveDb db2 = new HiveDb("db2", Lists.newArrayList(tab21));
- HiveDb db1 = new HiveDb("db1");
- HiveSnapshot snap = new HiveSnapshot().add(db1).add(db2).add(db3);
- MockClient mockClient = new MockClient(snap, 1);
- Mockito.when(mockClient.client.getCurrentNotificationEventId()).
- thenReturn(new CurrentNotificationEventId(mockClient.eventId));
- client.setClient(mockClient.client);
- hiveConnectionFactory.setClient(mockClient);
- // Make sure that client is connected
- Assert.assertTrue(client.isConnected());
-
- PathsImage snapshotInfo = client.getFullSnapshot();
- Assert.assertEquals(5, snapshotInfo.getPathImage().size());
- Assert.assertEquals(Sets.newHashSet("db1"), snapshotInfo.getPathImage().get("db1"));
- Assert.assertEquals(Sets.newHashSet("db2"), snapshotInfo.getPathImage().get("db2"));
- Assert.assertEquals(Sets.newHashSet("db3"), snapshotInfo.getPathImage().get("db3"));
- Assert.assertEquals(Sets.newHashSet("db2/tab21"),
- snapshotInfo.getPathImage().get("db2.tab21"));
- Assert.assertEquals(Sets.newHashSet("db3/tab31"), snapshotInfo.getPathImage().get("db3.tab31"));
- Assert.assertEquals(snapshotInfo.getId(), mockClient.eventId);
-
- }
-
- /**
- * Test scenario when there is no HMS connection
- * Getting new notifications
- */
- @Test
- public void testGetNewNotificationsWithOutClientConnected() throws Exception {
- HiveTable tab21 = new HiveTable("tab21");
- HiveTable tab31 = new HiveTable("tab31");
- HiveDb db3 = new HiveDb("db3", Lists.newArrayList(tab31));
- HiveDb db2 = new HiveDb("db2", Lists.newArrayList(tab21));
- HiveDb db1 = new HiveDb("db1");
- client.setClient(null);
- HiveSnapshot snap = new HiveSnapshot().add(db1).add(db2).add(db3);
- MockClient mockClient = new MockClient(snap, 100);
- Mockito.when(mockClient.client.getCurrentNotificationEventId()).
- thenReturn(new CurrentNotificationEventId(mockClient.eventId));
- // Make sure that client is not connected
- Assert.assertTrue(!client.isConnected());
- Collection<NotificationEvent> events = client.getNotifications(100);
- Assert.assertTrue(events.isEmpty());
-
- }
-
- /**
- * Test scenario where there are no notifications
- * Getting new notifications
- */
- @Test
- public void testGetNewNotificationsWithNoHmsUpdates() throws Exception {
- HiveTable tab21 = new HiveTable("tab21");
- HiveTable tab31 = new HiveTable("tab31");
- HiveDb db3 = new HiveDb("db3", Lists.newArrayList(tab31));
- HiveDb db2 = new HiveDb("db2", Lists.newArrayList(tab21));
- HiveDb db1 = new HiveDb("db1");
- HiveSnapshot snap = new HiveSnapshot().add(db1).add(db2).add(db3);
- MockClient mockClient = new MockClient(snap, 100);
- Mockito.when(mockClient.client.getCurrentNotificationEventId()).
- thenReturn(new CurrentNotificationEventId(mockClient.eventId));
- client.setClient(mockClient.client);
- hiveConnectionFactory.setClient(mockClient);
- // Make sure that client is connected
- Assert.assertTrue(client.isConnected());
- Collection<NotificationEvent> events = client.getNotifications(100);
- Assert.assertTrue(events.isEmpty());
- }
-
- /**
- * Test scenario where there are notifications
- * Getting new notifications
- */
- @Test
- public void testGetNewNotificationsSuccess() throws Exception {
- final MockClient mockClient = new MockClient(new HiveSnapshot(), 100);
- client.setClient(mockClient.client);
- hiveConnectionFactory.setClient(mockClient);
- // Make sure that client is connected
- Assert.assertTrue(client.isConnected());
-
- Mockito.when(mockClient.client.getCurrentNotificationEventId()).
- thenAnswer(new Answer<CurrentNotificationEventId>() {
- @Override
- public CurrentNotificationEventId answer(InvocationOnMock invocation)
- throws Throwable {
- return new CurrentNotificationEventId(mockClient.incrementNotificationEventId());
- }
- });
- Mockito.when(mockClient.client.getNextNotification(Mockito.anyLong(), Mockito.anyInt(),
- Mockito.any(NotificationFilter.class))).
- thenAnswer(new Answer<NotificationEventResponse>() {
- @Override
- public NotificationEventResponse answer(InvocationOnMock invocation)
- throws Throwable {
- long id = 1;
- List<NotificationEvent> events = new ArrayList<>();
- events.add(getCreateDatabaseNotification(id++));
- events.add(getDropDatabaseNotification(id++));
- return new NotificationEventResponse(events);
- }
- });
-
- Collection<NotificationEvent> events = client.getNotifications(100);
- long id = 1;
- for (NotificationEvent event : events) {
- Assert.assertEquals(event.getEventId(), id++);
- }
- Assert.assertTrue(events.size() == 2);
- }
-
- /**
- * Representation of a Hive table. A table has a name and a list of partitions.
- */
- private static class HiveTable {
-
- private final String name;
- private final List<String> partitions;
-
- HiveTable(String name) {
- this.name = name;
- this.partitions = new ArrayList<>();
- }
-
- HiveTable add(String partition) {
- partitions.add(partition);
- return this;
- }
- }
-
- /**
- * Representation of a Hive database. A database has a name and a list of tables
- */
- private static class HiveDb {
-
- final String name;
- Collection<HiveTable> tables;
-
- @SuppressWarnings("SameParameterValue")
- HiveDb(String name) {
- this.name = name;
- tables = new ArrayList<>();
- }
-
- HiveDb(String name, Collection<HiveTable> tables) {
- this.name = name;
- this.tables = tables;
- if (this.tables == null) {
- this.tables = new ArrayList<>();
- }
- }
-
- void add(HiveTable table) {
- this.tables.add(table);
- }
- }
-
- /**
- * Representation of a full Hive snapshot. A snapshot is collection of databases
- */
- private static class HiveSnapshot {
-
- final List<HiveDb> databases = new ArrayList<>();
-
- HiveSnapshot() {
- }
-
- HiveSnapshot(Collection<HiveDb> dblist) {
- if (dblist != null) {
- databases.addAll(dblist);
- }
- }
-
- HiveSnapshot add(HiveDb db) {
- this.databases.add(db);
- return this;
- }
- }
-
- /**
- * Mock for HMSClientFactory
- */
- private static class MockHMSClientFactory implements HiveConnectionFactory {
-
- private HiveMetaStoreClient mClient;
-
- public MockHMSClientFactory() {
- mClient = null;
- }
-
- void setClient(MockClient mockClient) {
- this.mClient = mockClient.client;
- }
- @Override
- public HMSClient connect() throws IOException, InterruptedException, MetaException {
- return new HMSClient(mClient);
- }
-
- @Override
- public void close() throws Exception {
- }
- }
-
- /**
- * Convert Hive snapshot to mock client that will return proper values
- * for the snapshot.
- */
- private static class MockClient {
-
- public HiveMetaStoreClient client;
- public long eventId;
-
- MockClient(HiveSnapshot snapshot, long eventId) throws TException {
- this.eventId = eventId;
- client = Mockito.mock(HiveMetaStoreClient.class);
- List<String> dbNames = new ArrayList<>(snapshot.databases.size());
- // Walk over all databases and mock appropriate objects
- for (HiveDb mdb : snapshot.databases) {
- String dbName = mdb.name;
- dbNames.add(dbName);
- Database db = makeDb(dbName);
- Mockito.when(client.getDatabase(dbName)).thenReturn(db);
- List<String> tableNames = new ArrayList<>(mdb.tables.size());
- // Walk over all tables for the database and mock appropriate objects
- for (HiveTable table : mdb.tables) {
- String tableName = table.name;
- tableNames.add(tableName);
- Table mockTable = makeTable(dbName, tableName);
- Mockito.when(client.getTableObjectsByName(dbName,
- Lists.newArrayList(tableName)))
- .thenReturn(Lists.newArrayList(mockTable));
- Mockito.when(client.listPartitionNames(dbName, tableName, (short) -1))
- .thenReturn(table.partitions);
- // Walk across all partitions and mock appropriate objects
- for (String partName : table.partitions) {
- Partition p = makePartition(dbName, tableName, partName);
- Mockito.when(client.getPartitionsByNames(dbName, tableName,
- Lists.<String>newArrayList(partName)))
- .thenReturn(Lists.<Partition>newArrayList(p));
- }
- }
- Mockito.when(client.getAllTables(dbName)).thenReturn(tableNames);
- }
- // Return all database names
- Mockito.when(client.getAllDatabases()).thenReturn(dbNames);
- Mockito.when(client.getCurrentNotificationEventId()).
- thenReturn(new CurrentNotificationEventId(eventId));
-
- }
-
- public Long incrementNotificationEventId() {
- eventId = eventId + 1;
- return eventId;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/sentry/blob/07d3ec17/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbPrivilegeCleanupOnDrop.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbPrivilegeCleanupOnDrop.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbPrivilegeCleanupOnDrop.java
index d619623..b9330cc 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbPrivilegeCleanupOnDrop.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbPrivilegeCleanupOnDrop.java
@@ -31,7 +31,7 @@ import java.sql.Statement;
import java.util.ArrayList;
import java.util.List;
-import org.apache.sentry.service.thrift.HmsFollower;
+import org.apache.sentry.service.thrift.HMSFollower;
import org.apache.sentry.tests.e2e.hive.AbstractTestWithStaticConfiguration;
import org.junit.BeforeClass;
import org.junit.Before;
@@ -78,7 +78,7 @@ public class TestDbPrivilegeCleanupOnDrop extends
to.close();
// Check the HMS connection only when notification log is enabled.
if (enableNotificationLog) {
- while (!HmsFollower.isConnectedToHms()) {
+ while (!HMSFollower.isConnectedToHMS()) {
Thread.sleep(1000);
}
}