You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sentry.apache.org by sr...@apache.org on 2016/07/13 20:45:58 UTC
[1/2] sentry git commit: SENTRY-1329: Adapt
SentryMetaStorePostEventListener to write HMS notification logs (Sravya
Tirukkovalur, Reviewed by: Hao Hao)
Repository: sentry
Updated Branches:
refs/heads/sentry-ha-redesign a70cff999 -> 73d77bcd0
http://git-wip-us.apache.org/repos/asf/sentry/blob/73d77bcd/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestDBNotificationListenerInBuiltDeserializer.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestDBNotificationListenerInBuiltDeserializer.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestDBNotificationListenerInBuiltDeserializer.java
new file mode 100644
index 0000000..56e19c4
--- /dev/null
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestDBNotificationListenerInBuiltDeserializer.java
@@ -0,0 +1,353 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.tests.e2e.metastore;
+
+import com.google.common.collect.Lists;
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.api.*;
+import org.apache.hive.hcatalog.messaging.CreateDatabaseMessage;
+import org.apache.hive.hcatalog.messaging.HCatEventMessage;
+import org.apache.hive.hcatalog.messaging.MessageDeserializer;
+import org.apache.hive.hcatalog.messaging.MessageFactory;
+import org.apache.hive.hcatalog.messaging.CreateTableMessage;
+import org.apache.hive.hcatalog.messaging.DropTableMessage;
+import org.apache.hive.hcatalog.messaging.AlterTableMessage;
+import org.apache.hive.hcatalog.messaging.AlterPartitionMessage;
+import org.apache.hive.hcatalog.messaging.DropDatabaseMessage;
+import org.apache.hive.hcatalog.messaging.AddPartitionMessage;
+import org.apache.hive.hcatalog.messaging.DropPartitionMessage;
+import org.apache.sentry.tests.e2e.hive.StaticUserGroup;
+import org.apache.sentry.tests.e2e.hive.hiveserver.HiveServerFactory;
+import org.hamcrest.text.IsEqualIgnoringCase;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
+
+import org.junit.*;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Random;
+
+/**
+ * Make sure NotificationLog is capturing the information correctly for the commands which change <Obj,Location> mapping
+ * This test class is using Hive's DbNotificationListener and Hive's Notification log JSON deserializer.
+ */
+
+public class TestDBNotificationListenerInBuiltDeserializer extends AbstractMetastoreTestWithStaticConfiguration {
+
+ protected static HiveMetaStoreClient client;
+ protected static MessageDeserializer deserializer;
+ protected static Random random = new Random();
+ private static String testDB;
+
+ @BeforeClass
+ public static void setupTestStaticConfiguration() throws Exception {
+ setMetastoreListener = true;
+ useDbNotificationListener = true;
+ beforeClass();
+ }
+
+ protected static void beforeClass() throws Exception {
+ AbstractMetastoreTestWithStaticConfiguration.setupTestStaticConfiguration();
+ client = context.getMetaStoreClient(ADMIN1);
+ deserializer = MessageFactory.getDeserializer("json", "");
+ writePolicyFile(setAdminOnServer1(ADMINGROUP).setUserGroupMapping(StaticUserGroup.getStaticMapping()));
+ }
+
+ @AfterClass
+ public static void cleanupAfterClass() throws Exception {
+ if (client != null) {
+ client.close();
+ }
+ }
+
+ @After
+ public void dropDBAfterTest() throws Exception {
+ if(client != null && testDB != null) {
+ dropMetastoreDBIfExists(client, testDB);
+ }
+ }
+
+ @Test
+ public void testCreateDropDatabase() throws Exception {
+ CurrentNotificationEventId latestID, previousID;
+ NotificationEventResponse response;
+
+ testDB = "N_db" + random.nextInt(Integer.SIZE - 1);
+
+ // Create database
+ // We need:
+ // - Dbname
+ // - location
+ createMetastoreDB(client, testDB);
+ latestID = client.getCurrentNotificationEventId();
+ response = client.getNextNotification(latestID.getEventId() - 1, 1, null);
+ CreateDatabaseMessage createDatabaseMessage = deserializer.getCreateDatabaseMessage(response.getEvents().get(0).getMessage());
+ assertEquals(HCatEventMessage.EventType.CREATE_DATABASE, createDatabaseMessage.getEventType()); //Validate EventType
+ assertEquals(testDB, createDatabaseMessage.getDB()); //dbName
+ //Location information is not available
+
+ //Alter database location and rename are not supported. See HIVE-4847
+
+ //Drop database
+ // We need:
+ // - dbName
+ // - location
+ client.dropDatabase(testDB);
+ previousID = latestID;
+ latestID = client.getCurrentNotificationEventId();
+ assertEquals(previousID.getEventId() + 1, latestID.getEventId()); //Validate monotonically increasing eventID
+ response = client.getNextNotification(latestID.getEventId() - 1, 1, null);
+ DropDatabaseMessage dropDatabaseMessage = deserializer.getDropDatabaseMessage(response.getEvents().get(0).getMessage());
+ assertEquals(HCatEventMessage.EventType.DROP_DATABASE, dropDatabaseMessage.getEventType()); //Event type
+ assertThat(dropDatabaseMessage.getDB(), IsEqualIgnoringCase.equalToIgnoringCase(testDB)); // dbName
+ //Location information is not available, but we might not really need it as we can drop all paths associated with
+ //the object when we drop
+ }
+
+ @Test
+ public void testCreateDropTableWithPartition() throws Exception {
+ testDB = "N_db" + random.nextInt(Integer.SIZE - 1);
+ String testTable = "N_table" + random.nextInt(Integer.SIZE - 1);
+
+ NotificationEventResponse response;
+ CurrentNotificationEventId latestID, previousID;
+ // Create database
+ createMetastoreDB(client, testDB);
+
+ // Create table with partition
+ // We need:
+ // - dbname
+ // - tablename
+ // - location
+ createMetastoreTableWithPartition(client, testDB,
+ testTable, Lists.newArrayList(new FieldSchema("col1", "int", "")),
+ Lists.newArrayList(new FieldSchema("part_col1", "string", "")));
+ latestID = client.getCurrentNotificationEventId();
+ response = client.getNextNotification(latestID.getEventId() - 1, 1, null);
+ CreateTableMessage createTableMessage = deserializer.getCreateTableMessage(response.getEvents().get(0).getMessage());
+ assertEquals(HCatEventMessage.EventType.CREATE_TABLE, createTableMessage.getEventType());
+ assertEquals(testDB, createTableMessage.getDB()); //dbName
+ assertEquals(testTable, createTableMessage.getTable()); //tableName
+ //Location information is not available
+
+ //Drop table
+ // We need:
+ // - dbName
+ // - tableName
+ // - location
+ client.dropTable(testDB, testTable);
+ previousID = latestID;
+ latestID = client.getCurrentNotificationEventId();
+ assertEquals(previousID.getEventId() + 1, latestID.getEventId());
+ response = client.getNextNotification(latestID.getEventId() - 1, 1, null);
+ DropTableMessage dropTableMessage = deserializer.getDropTableMessage(response.getEvents().get(0).getMessage());
+ assertEquals(HCatEventMessage.EventType.DROP_TABLE, dropTableMessage.getEventType());
+ assertThat(dropTableMessage.getDB(), IsEqualIgnoringCase.equalToIgnoringCase(testDB));//dbName
+ assertThat(dropTableMessage.getTable(), IsEqualIgnoringCase.equalToIgnoringCase(testTable));//tableName
+ //Location information is not available, but we might not really need it as we can drop all paths associated with
+ //the object when we drop
+ }
+
+ @Test
+ public void testCreateDropTableWithoutPartition() throws Exception {
+ testDB = "N_db" + random.nextInt(Integer.SIZE - 1);
+ String testTable = "N_table" + random.nextInt(Integer.SIZE - 1);
+
+ NotificationEventResponse response;
+ CurrentNotificationEventId latestID, previousID;
+ // Create database
+ createMetastoreDB(client, testDB);
+
+ // Create table with partition
+ // We need:
+ // - dbname
+ // - tablename
+ // - location
+ createMetastoreTable(client, testDB, testTable, Lists.newArrayList(new FieldSchema("col1", "int", "")));
+ latestID = client.getCurrentNotificationEventId();
+ response = client.getNextNotification(latestID.getEventId() - 1, 1, null);
+ CreateTableMessage createTableMessage = deserializer.getCreateTableMessage(response.getEvents().get(0).getMessage());
+ assertEquals(HCatEventMessage.EventType.CREATE_TABLE, createTableMessage.getEventType());
+ assertEquals(testDB, createTableMessage.getDB()); //dbName
+ assertEquals(testTable, createTableMessage.getTable()); //tableName
+ //Location information is not available
+
+ //Drop table
+ // We need:
+ // - dbName
+ // - tableName
+ // - location
+ client.dropTable(testDB, testTable);
+ previousID = latestID;
+ latestID = client.getCurrentNotificationEventId();
+ assertEquals(previousID.getEventId() + 1, latestID.getEventId());
+ response = client.getNextNotification(latestID.getEventId() - 1, 1, null);
+ DropTableMessage dropTableMessage = deserializer.getDropTableMessage(response.getEvents().get(0).getMessage());
+ assertEquals(HCatEventMessage.EventType.DROP_TABLE, dropTableMessage.getEventType());
+ assertThat(dropTableMessage.getDB(), IsEqualIgnoringCase.equalToIgnoringCase(testDB));//dbName
+ assertThat(dropTableMessage.getTable(), IsEqualIgnoringCase.equalToIgnoringCase(testTable));//tableName
+ //Location information is not available, but we might not really need it as we can drop all paths associated with
+ //the object when we drop
+ }
+
+ @Test
+ public void testAddDropPartition() throws Exception {
+ testDB = "N_db" + random.nextInt(Integer.SIZE - 1);
+ String testTable = "N_table" + random.nextInt(Integer.SIZE - 1);
+
+ NotificationEventResponse response;
+ CurrentNotificationEventId latestID, previousID;
+ // Create database and table
+ createMetastoreDB(client, testDB);
+ Table tbl1 = createMetastoreTableWithPartition(client, testDB, testTable, Lists.newArrayList(new FieldSchema("col1", "int", "")),
+ Lists.newArrayList(new FieldSchema("part_col1", "string", "")));
+
+ ArrayList<String> partVals1 = Lists.newArrayList("part1");
+
+ //Add partition
+ // We need:
+ // - dbName
+ // - tableName
+ // - partition location
+ addPartition(client, testDB, testTable, partVals1, tbl1);
+ latestID = client.getCurrentNotificationEventId();
+ response = client.getNextNotification(latestID.getEventId() - 1, 1, null);
+ AddPartitionMessage addPartitionMessage = deserializer.getAddPartitionMessage(response.getEvents().get(0).getMessage());
+ assertEquals(HCatEventMessage.EventType.ADD_PARTITION, addPartitionMessage.getEventType());
+ assertThat(addPartitionMessage.getDB(), IsEqualIgnoringCase.equalToIgnoringCase(testDB));// dbName (returns lowered version)
+ assertThat(addPartitionMessage.getTable(), IsEqualIgnoringCase.equalToIgnoringCase(testTable));// tableName (returns lowered version)
+ //Location information is not available
+
+ //Drop partition
+ // We need:
+ // - dbName
+ // - tableName
+ // - partition location
+ dropPartition(client, testDB, testTable, partVals1);
+ previousID = latestID;
+ latestID = client.getCurrentNotificationEventId();
+ assertEquals(previousID.getEventId() + 1, latestID.getEventId());
+ response = client.getNextNotification(latestID.getEventId() - 1, 1, null);
+ DropPartitionMessage dropPartitionMessage = deserializer.getDropPartitionMessage(response.getEvents().get(0).getMessage());
+ assertEquals(HCatEventMessage.EventType.DROP_PARTITION, dropPartitionMessage.getEventType());
+ assertThat(dropPartitionMessage.getDB(), IsEqualIgnoringCase.equalToIgnoringCase(testDB)); //dbName
+ assertThat(dropPartitionMessage.getTable(), IsEqualIgnoringCase.equalToIgnoringCase(testTable)); //tableName
+ //Location information is not available
+
+ }
+
+ @Ignore("Needs Hive >= 1.1.2")
+ @Test
+ public void testAlterTableWithPartition() throws Exception {
+ testDB = "N_db" + random.nextInt(Integer.SIZE - 1);
+ String testTable = "N_table" + random.nextInt(Integer.SIZE - 1);
+
+ NotificationEventResponse response;
+ CurrentNotificationEventId latestID, previousID;
+ // Create database
+ createMetastoreDB(client, testDB);
+
+ // Create table with partition
+ Table tbl1 = createMetastoreTableWithPartition(client, testDB,
+ testTable, Lists.newArrayList(new FieldSchema("col1", "int", "")),
+ Lists.newArrayList(new FieldSchema("part_col1", "string", "")));
+
+ //Alter table location
+ // We need:
+ // - dbName
+ // - tableName
+ // - old location
+ // - new location
+ String tabDir1 = hiveServer.getProperty(HiveServerFactory.WAREHOUSE_DIR)
+ + File.separator + random.nextInt(Integer.SIZE - 1);
+ alterTableWithLocation(client, tbl1, tabDir1);
+ latestID = client.getCurrentNotificationEventId();
+ response = client.getNextNotification(latestID.getEventId()-1, 1, null);
+ AlterTableMessage alterTableMessage = deserializer.getAlterTableMessage(response.getEvents().get(0).getMessage());
+ assertEquals(HCatEventMessage.EventType.ALTER_TABLE, alterTableMessage.getEventType());
+ assertThat(alterTableMessage.getDB(), IsEqualIgnoringCase.equalToIgnoringCase(testDB));//dbName
+ assertThat(alterTableMessage.getTable(), IsEqualIgnoringCase.equalToIgnoringCase(testTable));//tableName
+ //Old location is not available: This information is lost if not captured at the time of event.
+ //New location is not available
+
+ //Alter table rename managed table - location also changes
+ // We need:
+ // - oldDbName
+ // - newDbName
+ // - oldTableName
+ // - newTableName
+ // - old location
+ // - new location
+ String newDBName = testDB + random.nextInt(Integer.SIZE - 1);
+ String newTableName = testTable + random.nextInt(Integer.SIZE - 1);
+ String newLocation = tabDir1 + random.nextInt(Integer.SIZE - 1);
+ createMetastoreDB(client, newDBName);
+ alterTableRename(client, tbl1, newDBName, newTableName, newLocation);
+ previousID = latestID;
+ latestID = client.getCurrentNotificationEventId();
+ assertEquals(previousID.getEventId() + 1, latestID.getEventId());
+ response = client.getNextNotification(latestID.getEventId()-1, 1, null);
+ alterTableMessage = deserializer.getAlterTableMessage(response.getEvents().get(0).getMessage());
+ assertEquals(HCatEventMessage.EventType.ALTER_TABLE, alterTableMessage.getEventType());
+ assertThat(alterTableMessage.getDB(), IsEqualIgnoringCase.equalToIgnoringCase(testDB));//oldDbName
+ assertThat(alterTableMessage.getTable(), IsEqualIgnoringCase.equalToIgnoringCase(testTable));//oldTableName
+ assertThat(response.getEvents().get(0).getDbName(), IsEqualIgnoringCase.equalToIgnoringCase(newDBName));//newDbName
+ assertThat(response.getEvents().get(0).getTableName(), IsEqualIgnoringCase.equalToIgnoringCase(newTableName));//newTableName
+ //Old location: This information is lost if not captured at the time of event.
+ //New location: Not sure how can we get this? Refresh all paths for every alter table add partition?
+ }
+
+ @Ignore("Needs Hive >= 1.1.2")
+ @Test
+ public void testAlterPartition() throws Exception {
+ testDB = "N_db" + random.nextInt(Integer.SIZE - 1);
+ String testTable = "N_table" + random.nextInt(Integer.SIZE - 1);
+
+ NotificationEventResponse response;
+ CurrentNotificationEventId latestID;
+ // Create database
+ createMetastoreDB(client, testDB);
+
+ // Create table with partition
+ Table tbl1 = createMetastoreTableWithPartition(client, testDB,
+ testTable, Lists.newArrayList(new FieldSchema("col1", "int", "")),
+ Lists.newArrayList(new FieldSchema("part_col1", "string", "")));
+ ArrayList<String> partVals1 = Lists.newArrayList("part1");
+ Partition partition = addPartition(client, testDB, testTable, partVals1, tbl1);
+
+
+ String warehouseDir = hiveServer.getProperty(HiveServerFactory.WAREHOUSE_DIR);
+ //Alter partition with location
+ // We need:
+ // - dbName
+ // - tableName
+ // - partition location
+ alterPartitionWithLocation(client, partition, warehouseDir + File.separator + "newpart");
+ latestID = client.getCurrentNotificationEventId();
+ response = client.getNextNotification(latestID.getEventId()-1, 1, null);
+ AlterPartitionMessage alterPartitionMessage = deserializer.getAlterPartitionMessage(response.getEvents().get(0).getMessage());
+ assertEquals(HCatEventMessage.EventType.ALTER_PARTITION, alterPartitionMessage.getEventType());
+ assertThat(alterPartitionMessage.getDB(), IsEqualIgnoringCase.equalToIgnoringCase(testDB));// dbName
+ assertThat(alterPartitionMessage.getTable(), IsEqualIgnoringCase.equalToIgnoringCase(testTable));// tableName
+ //Location information, not sure how can we get this? Refresh all paths for every alter table add partition?
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/sentry/blob/73d77bcd/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestDbNotificationListenerSentryDeserializer.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestDbNotificationListenerSentryDeserializer.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestDbNotificationListenerSentryDeserializer.java
new file mode 100644
index 0000000..8e588b1
--- /dev/null
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestDbNotificationListenerSentryDeserializer.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.tests.e2e.metastore;
+
+import org.junit.BeforeClass;
+
+
+/**
+ * Make sure we are able to capture all HMS object and path changes using Hive's DbNotificationListener and
+ * Sentry's JSON deserializer. This would make sure Sentry is able to read the Notification logs written by
+ * Hive's DBNotificationListener
+ */
+public class TestDbNotificationListenerSentryDeserializer extends TestSentryListenerSentryDeserializer {
+
+ @BeforeClass
+ public static void setupTestStaticConfiguration() throws Exception {
+ setMetastoreListener = true;
+ useDbNotificationListener = true;
+ AbstractMetastoreTestWithStaticConfiguration.setupTestStaticConfiguration();
+ setupClass();
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/sentry/blob/73d77bcd/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestHMSNotificationLogUsingDBNotificationListener.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestHMSNotificationLogUsingDBNotificationListener.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestHMSNotificationLogUsingDBNotificationListener.java
deleted file mode 100644
index 0b328d4..0000000
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestHMSNotificationLogUsingDBNotificationListener.java
+++ /dev/null
@@ -1,351 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.sentry.tests.e2e.metastore;
-
-import com.google.common.collect.Lists;
-import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
-import org.apache.hadoop.hive.metastore.api.*;
-import org.apache.hive.hcatalog.messaging.CreateDatabaseMessage;
-import org.apache.hive.hcatalog.messaging.HCatEventMessage;
-import org.apache.hive.hcatalog.messaging.MessageDeserializer;
-import org.apache.hive.hcatalog.messaging.MessageFactory;
-import org.apache.hive.hcatalog.messaging.CreateTableMessage;
-import org.apache.hive.hcatalog.messaging.DropTableMessage;
-import org.apache.hive.hcatalog.messaging.AlterTableMessage;
-import org.apache.hive.hcatalog.messaging.AlterPartitionMessage;
-import org.apache.hive.hcatalog.messaging.DropDatabaseMessage;
-import org.apache.hive.hcatalog.messaging.AddPartitionMessage;
-import org.apache.hive.hcatalog.messaging.DropPartitionMessage;
-import org.apache.sentry.provider.file.PolicyFile;
-import org.apache.sentry.tests.e2e.hive.StaticUserGroup;
-import org.apache.sentry.tests.e2e.hive.hiveserver.HiveServerFactory;
-import org.hamcrest.text.IsEqualIgnoringCase;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertThat;
-
-import org.junit.*;
-
-import java.io.File;
-import java.util.ArrayList;
-import java.util.Random;
-
-/**
- * Make sure NotificationLog is capturing the information correctly for the commands which change <Obj,Location> mapping
- */
-public class TestHMSNotificationLogUsingDBNotificationListener extends AbstractMetastoreTestWithStaticConfiguration {
-
- private PolicyFile policyFile;
-
- private static HiveMetaStoreClient client;
- private static MessageDeserializer deserializer;
- private static Random random = new Random();
-
-
- @BeforeClass
- public static void setupTestStaticConfiguration() throws Exception {
- setMetastoreListener = true;
- useDbNotificationListener = true;
- AbstractMetastoreTestWithStaticConfiguration.setupTestStaticConfiguration();
- client = context.getMetaStoreClient(ADMIN1);
- deserializer = MessageFactory.getDeserializer("json", "");
- }
-
- @AfterClass
- public static void cleanupAfterClass() throws Exception {
- if (client != null) {
- client.close();
- }
- }
-
- @Override
- @Before
- public void setup() throws Exception {
- policyFile = setAdminOnServer1(ADMINGROUP);
- policyFile.setUserGroupMapping(StaticUserGroup.getStaticMapping());
- writePolicyFile(policyFile);
- super.setup();
- }
-
- @Test
- public void testCreateDropDatabase() throws Exception {
- CurrentNotificationEventId latestID, previousID;
- NotificationEventResponse response;
-
- String testDB = "N_db" + random.nextInt(Integer.SIZE - 1);
-
- // Create database
- // We need:
- // - Dbname
- // - location
- createMetastoreDB(client, testDB);
- latestID = client.getCurrentNotificationEventId();
- response = client.getNextNotification(latestID.getEventId() - 1, 1, null);
- CreateDatabaseMessage createDatabaseMessage = deserializer.getCreateDatabaseMessage(response.getEvents().get(0).getMessage());
- assertEquals(HCatEventMessage.EventType.CREATE_DATABASE, createDatabaseMessage.getEventType()); //Validate EventType
- assertEquals(testDB, createDatabaseMessage.getDB()); //dbName
- //Location information is not available
-
- //Alter database location and rename are not supported. See HIVE-4847
-
- //Drop database
- // We need:
- // - dbName
- // - location
- client.dropDatabase(testDB);
- previousID = latestID;
- latestID = client.getCurrentNotificationEventId();
- assertEquals(previousID.getEventId() + 1, latestID.getEventId()); //Validate monotonically increasing eventID
- response = client.getNextNotification(latestID.getEventId() - 1, 1, null);
- DropDatabaseMessage dropDatabaseMessage = deserializer.getDropDatabaseMessage(response.getEvents().get(0).getMessage());
- assertEquals(HCatEventMessage.EventType.DROP_DATABASE, dropDatabaseMessage.getEventType()); //Event type
- assertThat(dropDatabaseMessage.getDB(), IsEqualIgnoringCase.equalToIgnoringCase(testDB)); // dbName
- //Location information is not available, but we might not really need it as we can drop all paths associated with
- //the object when we drop
- }
-
- @Test
- public void testCreateDropTableWithPartition() throws Exception {
- String testDB = "N_db" + random.nextInt(Integer.SIZE - 1);
- String testTable = "N_table" + random.nextInt(Integer.SIZE - 1);
-
- NotificationEventResponse response;
- CurrentNotificationEventId latestID, previousID;
- // Create database
- createMetastoreDB(client, testDB);
-
- // Create table with partition
- // We need:
- // - dbname
- // - tablename
- // - location
- createMetastoreTableWithPartition(client, testDB,
- testTable, Lists.newArrayList(new FieldSchema("col1", "int", "")),
- Lists.newArrayList(new FieldSchema("part_col1", "string", "")));
- latestID = client.getCurrentNotificationEventId();
- response = client.getNextNotification(latestID.getEventId() - 1, 1, null);
- CreateTableMessage createTableMessage = deserializer.getCreateTableMessage(response.getEvents().get(0).getMessage());
- assertEquals(HCatEventMessage.EventType.CREATE_TABLE, createTableMessage.getEventType());
- assertEquals(testDB, createTableMessage.getDB()); //dbName
- assertEquals(testTable, createTableMessage.getTable()); //tableName
- //Location information is not available
-
- //Drop table
- // We need:
- // - dbName
- // - tableName
- // - location
- client.dropTable(testDB, testTable);
- previousID = latestID;
- latestID = client.getCurrentNotificationEventId();
- assertEquals(previousID.getEventId() + 1, latestID.getEventId());
- response = client.getNextNotification(latestID.getEventId() - 1, 1, null);
- DropTableMessage dropTableMessage = deserializer.getDropTableMessage(response.getEvents().get(0).getMessage());
- assertEquals(HCatEventMessage.EventType.DROP_TABLE, dropTableMessage.getEventType());
- assertThat(dropTableMessage.getDB(), IsEqualIgnoringCase.equalToIgnoringCase(testDB));//dbName
- assertThat(dropTableMessage.getTable(), IsEqualIgnoringCase.equalToIgnoringCase(testTable));//tableName
- //Location information is not available, but we might not really need it as we can drop all paths associated with
- //the object when we drop
- }
-
- @Test
- public void testCreateDropTableWithoutPartition() throws Exception {
- String testDB = "N_db" + random.nextInt(Integer.SIZE - 1);
- String testTable = "N_table" + random.nextInt(Integer.SIZE - 1);
-
- NotificationEventResponse response;
- CurrentNotificationEventId latestID, previousID;
- // Create database
- createMetastoreDB(client, testDB);
-
- // Create table with partition
- // We need:
- // - dbname
- // - tablename
- // - location
- createMetastoreTable(client, testDB, testTable, Lists.newArrayList(new FieldSchema("col1", "int", "")));
- latestID = client.getCurrentNotificationEventId();
- response = client.getNextNotification(latestID.getEventId() - 1, 1, null);
- CreateTableMessage createTableMessage = deserializer.getCreateTableMessage(response.getEvents().get(0).getMessage());
- assertEquals(HCatEventMessage.EventType.CREATE_TABLE, createTableMessage.getEventType());
- assertEquals(testDB, createTableMessage.getDB()); //dbName
- assertEquals(testTable, createTableMessage.getTable()); //tableName
- //Location information is not available
-
- //Drop table
- // We need:
- // - dbName
- // - tableName
- // - location
- client.dropTable(testDB, testTable);
- previousID = latestID;
- latestID = client.getCurrentNotificationEventId();
- assertEquals(previousID.getEventId() + 1, latestID.getEventId());
- response = client.getNextNotification(latestID.getEventId() - 1, 1, null);
- DropTableMessage dropTableMessage = deserializer.getDropTableMessage(response.getEvents().get(0).getMessage());
- assertEquals(HCatEventMessage.EventType.DROP_TABLE, dropTableMessage.getEventType());
- assertThat(dropTableMessage.getDB(), IsEqualIgnoringCase.equalToIgnoringCase(testDB));//dbName
- assertThat(dropTableMessage.getTable(), IsEqualIgnoringCase.equalToIgnoringCase(testTable));//tableName
- //Location information is not available, but we might not really need it as we can drop all paths associated with
- //the object when we drop
- }
-
- @Test
- public void testAddDropPartition() throws Exception {
- String testDB = "N_db" + random.nextInt(Integer.SIZE - 1);
- String testTable = "N_table" + random.nextInt(Integer.SIZE - 1);
-
- NotificationEventResponse response;
- CurrentNotificationEventId latestID, previousID;
- // Create database and table
- createMetastoreDB(client, testDB);
- Table tbl1 = createMetastoreTableWithPartition(client, testDB, testTable, Lists.newArrayList(new FieldSchema("col1", "int", "")),
- Lists.newArrayList(new FieldSchema("part_col1", "string", "")));
-
- ArrayList<String> partVals1 = Lists.newArrayList("part1");
-
- //Add partition
- // We need:
- // - dbName
- // - tableName
- // - partition location
- addPartition(client, testDB, testTable, partVals1, tbl1);
- latestID = client.getCurrentNotificationEventId();
- response = client.getNextNotification(latestID.getEventId() - 1, 1, null);
- AddPartitionMessage addPartitionMessage = deserializer.getAddPartitionMessage(response.getEvents().get(0).getMessage());
- assertEquals(HCatEventMessage.EventType.ADD_PARTITION, addPartitionMessage.getEventType());
- assertThat(addPartitionMessage.getDB(), IsEqualIgnoringCase.equalToIgnoringCase(testDB));// dbName (returns lowered version)
- assertThat(addPartitionMessage.getTable(), IsEqualIgnoringCase.equalToIgnoringCase(testTable));// tableName (returns lowered version)
- //Location information is not available
-
- //Drop partition
- // We need:
- // - dbName
- // - tableName
- // - partition location
- dropPartition(client, testDB, testTable, partVals1);
- previousID = latestID;
- latestID = client.getCurrentNotificationEventId();
- assertEquals(previousID.getEventId() + 1, latestID.getEventId());
- response = client.getNextNotification(latestID.getEventId() - 1, 1, null);
- DropPartitionMessage dropPartitionMessage = deserializer.getDropPartitionMessage(response.getEvents().get(0).getMessage());
- assertEquals(HCatEventMessage.EventType.DROP_PARTITION, dropPartitionMessage.getEventType());
- assertThat(dropPartitionMessage.getDB(), IsEqualIgnoringCase.equalToIgnoringCase(testDB)); //dbName
- assertThat(dropPartitionMessage.getTable(), IsEqualIgnoringCase.equalToIgnoringCase(testTable)); //tableName
- //Location information is not available
-
- }
-
- @Ignore("Needs Hive >= 1.1.2")
- @Test
- public void testAlterTableWithPartition() throws Exception {
- String testDB = "N_db" + random.nextInt(Integer.SIZE - 1);
- String testTable = "N_table" + random.nextInt(Integer.SIZE - 1);
-
- NotificationEventResponse response;
- CurrentNotificationEventId latestID, previousID;
- // Create database
- createMetastoreDB(client, testDB);
-
- // Create table with partition
- Table tbl1 = createMetastoreTableWithPartition(client, testDB,
- testTable, Lists.newArrayList(new FieldSchema("col1", "int", "")),
- Lists.newArrayList(new FieldSchema("part_col1", "string", "")));
-
- //Alter table location
- // We need:
- // - dbName
- // - tableName
- // - old location
- // - new location
- String tabDir1 = hiveServer.getProperty(HiveServerFactory.WAREHOUSE_DIR)
- + File.separator + random.nextInt(Integer.SIZE - 1);
- alterTableWithLocation(client, tbl1, tabDir1);
- latestID = client.getCurrentNotificationEventId();
- response = client.getNextNotification(latestID.getEventId()-1, 1, null);
- AlterTableMessage alterTableMessage = deserializer.getAlterTableMessage(response.getEvents().get(0).getMessage());
- assertEquals(HCatEventMessage.EventType.ALTER_TABLE, alterTableMessage.getEventType());
- assertThat(alterTableMessage.getDB(), IsEqualIgnoringCase.equalToIgnoringCase(testDB));//dbName
- assertThat(alterTableMessage.getTable(), IsEqualIgnoringCase.equalToIgnoringCase(testTable));//tableName
- //Old location is not available: This information is lost if not captured at the time of event.
- //New location is not available
-
- //Alter table rename managed table - location also changes
- // We need:
- // - oldDbName
- // - newDbName
- // - oldTableName
- // - newTableName
- // - old location
- // - new location
- String newDBName = testDB + random.nextInt(Integer.SIZE - 1);
- String newTableName = testTable + random.nextInt(Integer.SIZE - 1);
- String newLocation = tabDir1 + random.nextInt(Integer.SIZE - 1);
- createMetastoreDB(client, newDBName);
- alterTableRename(client, tbl1, newDBName, newTableName, newLocation);
- previousID = latestID;
- latestID = client.getCurrentNotificationEventId();
- assertEquals(previousID.getEventId() + 1, latestID.getEventId());
- response = client.getNextNotification(latestID.getEventId()-1, 1, null);
- alterTableMessage = deserializer.getAlterTableMessage(response.getEvents().get(0).getMessage());
- assertEquals(HCatEventMessage.EventType.ALTER_TABLE, alterTableMessage.getEventType());
- assertThat(alterTableMessage.getDB(), IsEqualIgnoringCase.equalToIgnoringCase(testDB));//oldDbName
- assertThat(alterTableMessage.getTable(), IsEqualIgnoringCase.equalToIgnoringCase(testTable));//oldTableName
- assertThat(response.getEvents().get(0).getDbName(), IsEqualIgnoringCase.equalToIgnoringCase(newDBName));//newDbName
- assertThat(response.getEvents().get(0).getTableName(), IsEqualIgnoringCase.equalToIgnoringCase(newTableName));//newTableName
- //Old location: This information is lost if not captured at the time of event.
- //New location: Not sure how can we get this? Refresh all paths for every alter table add partition?
- }
-
- @Ignore("Needs Hive >= 1.1.2")
- @Test
- public void testAlterPartition() throws Exception {
- String testDB = "N_db" + random.nextInt(Integer.SIZE - 1);
- String testTable = "N_table" + random.nextInt(Integer.SIZE - 1);
-
- NotificationEventResponse response;
- CurrentNotificationEventId latestID;
- // Create database
- createMetastoreDB(client, testDB);
-
- // Create table with partition
- Table tbl1 = createMetastoreTableWithPartition(client, testDB,
- testTable, Lists.newArrayList(new FieldSchema("col1", "int", "")),
- Lists.newArrayList(new FieldSchema("part_col1", "string", "")));
- ArrayList<String> partVals1 = Lists.newArrayList("part1");
- Partition partition = addPartition(client, testDB, testTable, partVals1, tbl1);
-
-
- String warehouseDir = hiveServer.getProperty(HiveServerFactory.WAREHOUSE_DIR);
- //Alter partition with location
- // We need:
- // - dbName
- // - tableName
- // - partition location
- alterPartitionWithLocation(client, partition, warehouseDir + File.separator + "newpart");
- latestID = client.getCurrentNotificationEventId();
- response = client.getNextNotification(latestID.getEventId()-1, 1, null);
- AlterPartitionMessage alterPartitionMessage = deserializer.getAlterPartitionMessage(response.getEvents().get(0).getMessage());
- assertEquals(HCatEventMessage.EventType.ALTER_PARTITION, alterPartitionMessage.getEventType());
- assertThat(alterPartitionMessage.getDB(), IsEqualIgnoringCase.equalToIgnoringCase(testDB));// dbName
- assertThat(alterPartitionMessage.getTable(), IsEqualIgnoringCase.equalToIgnoringCase(testTable));// tableName
- //Location information, not sure how can we get this? Refresh all paths for every alter table add partition?
- }
-}
-
http://git-wip-us.apache.org/repos/asf/sentry/blob/73d77bcd/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestSentryListenerInBuiltDeserializer.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestSentryListenerInBuiltDeserializer.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestSentryListenerInBuiltDeserializer.java
new file mode 100644
index 0000000..c4be62d
--- /dev/null
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestSentryListenerInBuiltDeserializer.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.tests.e2e.metastore;
+
+import org.junit.BeforeClass;
+
+/**
+ * Make sure we are able to capture all HMS object and path changes using Sentry's SentryMetastorePostEventListener
+ * and Hive's inbuilt Notification log deserializer. This would make sure Sentry is not breaking other users of
+ * NotificationLog who might be using Hive's in built serializer
+ */
+public class TestSentryListenerInBuiltDeserializer extends TestDBNotificationListenerInBuiltDeserializer {
+
+ @BeforeClass
+ public static void setupTestStaticConfiguration() throws Exception {
+ setMetastoreListener = true;
+ useDbNotificationListener = false;
+ beforeClass();
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/sentry/blob/73d77bcd/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestSentryListenerSentryDeserializer.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestSentryListenerSentryDeserializer.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestSentryListenerSentryDeserializer.java
new file mode 100644
index 0000000..6f1886f
--- /dev/null
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestSentryListenerSentryDeserializer.java
@@ -0,0 +1,375 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.tests.e2e.metastore;
+
+import com.google.common.collect.Lists;
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.api.*;
+import org.apache.hive.hcatalog.messaging.HCatEventMessage;
+import org.apache.sentry.binding.metastore.messaging.json.*;
+import org.apache.sentry.tests.e2e.hive.StaticUserGroup;
+import org.apache.sentry.tests.e2e.hive.hiveserver.HiveServerFactory;
+import org.hamcrest.text.IsEqualIgnoringCase;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
+
+import org.junit.*;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Random;
+
+/**
+ * Make sure we are able to capture all HMS object and path changes using Sentry's SentryMetastorePostEventListener
+ * and Sentry Notification log deserializer. Can be removed if we move to using DBNotificationListener
+ */
+public class TestSentryListenerSentryDeserializer extends AbstractMetastoreTestWithStaticConfiguration {
+
+ protected static HiveMetaStoreClient client;
+ protected static SentryJSONMessageDeserializer deserializer;
+ protected static Random random = new Random();
+ private static String warehouseDir;
+ private static String testDB;
+
+
+ @BeforeClass
+ public static void setupTestStaticConfiguration() throws Exception {
+ setMetastoreListener = true;
+ useDbNotificationListener = false;
+ AbstractMetastoreTestWithStaticConfiguration.setupTestStaticConfiguration();
+ setupClass();
+ }
+
+ protected static void setupClass() throws Exception{
+ client = context.getMetaStoreClient(ADMIN1);
+ deserializer = new SentryJSONMessageDeserializer();
+ warehouseDir = hiveServer.getProperty(HiveServerFactory.WAREHOUSE_DIR);
+ writePolicyFile(setAdminOnServer1(ADMINGROUP).setUserGroupMapping(StaticUserGroup.getStaticMapping()));
+
+ }
+
+ @AfterClass
+ public static void cleanupAfterClass() throws Exception {
+ if (client != null) {
+ client.close();
+ }
+ }
+
+ @After
+ public void dropDBAfterTest() throws Exception {
+ if(client != null && testDB != null) {
+ dropMetastoreDBIfExists(client, testDB);
+ }
+ }
+
+ @Test
+ public void testCreateDropDatabase() throws Exception {
+ CurrentNotificationEventId latestID, previousID;
+ NotificationEventResponse response;
+
+ testDB = "N_db" + random.nextInt(Integer.SIZE - 1);
+
+ // Create database
+ // We need:
+ // - Dbname
+ // - location
+ createMetastoreDB(client, testDB);
+ latestID = client.getCurrentNotificationEventId();
+ response = client.getNextNotification(latestID.getEventId() - 1, 1, null);
+ SentryJSONCreateDatabaseMessage createDatabaseMessage = deserializer.getCreateDatabaseMessage(response.getEvents().get(0).getMessage());
+ assertEquals(HCatEventMessage.EventType.CREATE_DATABASE, createDatabaseMessage.getEventType()); //Validate EventType
+ assertEquals(testDB, createDatabaseMessage.getDB()); //dbName
+ String expectedLocation = warehouseDir + "/" + testDB + ".db";
+ if(!useDbNotificationListener) {
+ Assert.assertEquals(expectedLocation.toLowerCase(), createDatabaseMessage.getLocation());
+ }
+
+ //Alter database location and rename are not supported. See HIVE-4847
+
+ //Drop database
+ // We need:
+ // - dbName
+ // - location
+ client.dropDatabase(testDB);
+ previousID = latestID;
+ latestID = client.getCurrentNotificationEventId();
+ assertEquals(previousID.getEventId() + 1, latestID.getEventId()); //Validate monotonically increasing eventID
+ response = client.getNextNotification(latestID.getEventId() - 1, 1, null);
+ SentryJSONDropDatabaseMessage dropDatabaseMessage = deserializer.getDropDatabaseMessage(response.getEvents().get(0).getMessage());
+ assertEquals(HCatEventMessage.EventType.DROP_DATABASE, dropDatabaseMessage.getEventType()); //Event type
+ assertThat(dropDatabaseMessage.getDB(), IsEqualIgnoringCase.equalToIgnoringCase(testDB)); // dbName
+ if(!useDbNotificationListener) {
+ Assert.assertEquals(expectedLocation.toLowerCase(), dropDatabaseMessage.getLocation()); //location
+ }
+ }
+
+ @Test
+ public void testCreateDropTableWithPartition() throws Exception {
+ testDB = "N_db" + random.nextInt(Integer.SIZE - 1);
+ String testTable = "N_table" + random.nextInt(Integer.SIZE - 1);
+
+ NotificationEventResponse response;
+ CurrentNotificationEventId latestID, previousID;
+ // Create database
+ createMetastoreDB(client, testDB);
+
+ // Create table with partition
+ // We need:
+ // - dbname
+ // - tablename
+ // - location
+ createMetastoreTableWithPartition(client, testDB,
+ testTable, Lists.newArrayList(new FieldSchema("col1", "int", "")),
+ Lists.newArrayList(new FieldSchema("part_col1", "string", "")));
+ latestID = client.getCurrentNotificationEventId();
+ response = client.getNextNotification(latestID.getEventId() - 1, 1, null);
+ SentryJSONCreateTableMessage createTableMessage = deserializer.getCreateTableMessage(response.getEvents().get(0).getMessage());
+ assertEquals(HCatEventMessage.EventType.CREATE_TABLE, createTableMessage.getEventType());
+ assertEquals(testDB, createTableMessage.getDB()); //dbName
+ assertEquals(testTable, createTableMessage.getTable()); //tableName
+ String expectedLocation = warehouseDir + "/" + testDB + ".db/" + testTable;
+ if(!useDbNotificationListener) {
+ Assert.assertEquals(expectedLocation.toLowerCase(), createTableMessage.getLocation());
+ }
+
+
+ //Drop table
+ // We need:
+ // - dbName
+ // - tableName
+ // - location
+ client.dropTable(testDB, testTable);
+ previousID = latestID;
+ latestID = client.getCurrentNotificationEventId();
+ assertEquals(previousID.getEventId() + 1, latestID.getEventId());
+ response = client.getNextNotification(latestID.getEventId() - 1, 1, null);
+ SentryJSONDropTableMessage dropTableMessage = deserializer.getDropTableMessage(response.getEvents().get(0).getMessage());
+ assertEquals(HCatEventMessage.EventType.DROP_TABLE, dropTableMessage.getEventType());
+ assertThat(dropTableMessage.getDB(), IsEqualIgnoringCase.equalToIgnoringCase(testDB));//dbName
+ assertThat(dropTableMessage.getTable(), IsEqualIgnoringCase.equalToIgnoringCase(testTable));//tableName
+ if(!useDbNotificationListener) {
+ Assert.assertEquals(expectedLocation.toLowerCase(), dropTableMessage.getLocation()); //location
+ }
+ }
+
+ @Test
+ public void testCreateDropTableWithoutPartition() throws Exception {
+ testDB = "N_db" + random.nextInt(Integer.SIZE - 1);
+ String testTable = "N_table" + random.nextInt(Integer.SIZE - 1);
+
+ NotificationEventResponse response;
+ CurrentNotificationEventId latestID, previousID;
+ // Create database
+ createMetastoreDB(client, testDB);
+
+ // Create table with partition
+ // We need:
+ // - dbname
+ // - tablename
+ // - location
+ createMetastoreTable(client, testDB, testTable, Lists.newArrayList(new FieldSchema("col1", "int", "")));
+ latestID = client.getCurrentNotificationEventId();
+ response = client.getNextNotification(latestID.getEventId() - 1, 1, null);
+ SentryJSONCreateTableMessage createTableMessage = deserializer.getCreateTableMessage(response.getEvents().get(0).getMessage());
+ assertEquals(HCatEventMessage.EventType.CREATE_TABLE, createTableMessage.getEventType());
+ assertEquals(testDB, createTableMessage.getDB()); //dbName
+ assertEquals(testTable, createTableMessage.getTable()); //tableName
+ String expectedLocation = warehouseDir + "/" + testDB + ".db/" + testTable;
+ if(!useDbNotificationListener) {
+ Assert.assertEquals(expectedLocation.toLowerCase(), createTableMessage.getLocation());
+ }
+
+ //Drop table
+ // We need:
+ // - dbName
+ // - tableName
+ // - location
+ client.dropTable(testDB, testTable);
+ previousID = latestID;
+ latestID = client.getCurrentNotificationEventId();
+ assertEquals(previousID.getEventId() + 1, latestID.getEventId());
+ response = client.getNextNotification(latestID.getEventId() - 1, 1, null);
+ SentryJSONDropTableMessage dropTableMessage = deserializer.getDropTableMessage(response.getEvents().get(0).getMessage());
+ assertEquals(HCatEventMessage.EventType.DROP_TABLE, dropTableMessage.getEventType());
+ assertThat(dropTableMessage.getDB(), IsEqualIgnoringCase.equalToIgnoringCase(testDB));//dbName
+ assertThat(dropTableMessage.getTable(), IsEqualIgnoringCase.equalToIgnoringCase(testTable));//tableName
+ if(!useDbNotificationListener) {
+ Assert.assertEquals(expectedLocation.toLowerCase(), dropTableMessage.getLocation()); //location
+ }
+ }
+
+ @Test
+ public void testAddDropPartition() throws Exception {
+ testDB = "N_db" + random.nextInt(Integer.SIZE - 1);
+ String testTable = "N_table" + random.nextInt(Integer.SIZE - 1);
+
+ NotificationEventResponse response;
+ CurrentNotificationEventId latestID, previousID;
+ // Create database and table
+ createMetastoreDB(client, testDB);
+ Table tbl1 = createMetastoreTableWithPartition(client, testDB, testTable, Lists.newArrayList(new FieldSchema("col1", "int", "")),
+ Lists.newArrayList(new FieldSchema("part_col1", "string", "")));
+
+ ArrayList<String> partVals1 = Lists.newArrayList("part1");
+
+ //Add partition
+ // We need:
+ // - dbName
+ // - tableName
+ // - partition location
+ addPartition(client, testDB, testTable, partVals1, tbl1);
+ latestID = client.getCurrentNotificationEventId();
+ response = client.getNextNotification(latestID.getEventId() - 1, 1, null);
+ SentryJSONAddPartitionMessage addPartitionMessage = deserializer.getAddPartitionMessage(response.getEvents().get(0).getMessage());
+ assertEquals(HCatEventMessage.EventType.ADD_PARTITION, addPartitionMessage.getEventType());
+ assertThat(addPartitionMessage.getDB(), IsEqualIgnoringCase.equalToIgnoringCase(testDB));// dbName (returns lowered version)
+ assertThat(addPartitionMessage.getTable(), IsEqualIgnoringCase.equalToIgnoringCase(testTable));// tableName (returns lowered version)
+ String expectedLocation = warehouseDir + "/" + testDB + ".db/" + testTable ; //TODO: SENTRY-1387: Tablelocation is stored instead of partition location
+ if(!useDbNotificationListener) {
+ Assert.assertEquals(expectedLocation.toLowerCase(), addPartitionMessage.getLocations().get(0));
+ }
+
+ //Drop partition
+ // We need:
+ // - dbName
+ // - tableName
+ // - partition location
+ dropPartition(client, testDB, testTable, partVals1);
+ previousID = latestID;
+ latestID = client.getCurrentNotificationEventId();
+ assertEquals(previousID.getEventId() + 1, latestID.getEventId());
+ response = client.getNextNotification(latestID.getEventId() - 1, 1, null);
+ SentryJSONDropPartitionMessage dropPartitionMessage = deserializer.getDropPartitionMessage(response.getEvents().get(0).getMessage());
+ assertEquals(HCatEventMessage.EventType.DROP_PARTITION, dropPartitionMessage.getEventType());
+ assertThat(dropPartitionMessage.getDB(), IsEqualIgnoringCase.equalToIgnoringCase(testDB)); //dbName
+ assertThat(dropPartitionMessage.getTable(), IsEqualIgnoringCase.equalToIgnoringCase(testTable)); //tableName
+ if(!useDbNotificationListener) {
+ Assert.assertEquals(expectedLocation.toLowerCase(), dropPartitionMessage.getLocation());
+ }
+
+ }
+
+ @Test
+ public void testAlterTableWithPartition() throws Exception {
+ testDB = "N_db" + random.nextInt(Integer.SIZE - 1);
+ String testTable = "N_table" + random.nextInt(Integer.SIZE - 1);
+
+ NotificationEventResponse response;
+ CurrentNotificationEventId latestID, previousID;
+ // Create database
+ createMetastoreDB(client, testDB);
+
+ // Create table with partition
+ Table tbl1 = createMetastoreTableWithPartition(client, testDB,
+ testTable, Lists.newArrayList(new FieldSchema("col1", "int", "")),
+ Lists.newArrayList(new FieldSchema("part_col1", "string", "")));
+ String oldLocation = tbl1.getSd().getLocation();
+
+ //Alter table location
+ // We need:
+ // - dbName
+ // - tableName
+ // - old location
+ // - new location
+ String tabDir1 = hiveServer.getProperty(HiveServerFactory.WAREHOUSE_DIR)
+ + File.separator + random.nextInt(Integer.SIZE - 1);
+ alterTableWithLocation(client, tbl1, tabDir1);
+ latestID = client.getCurrentNotificationEventId();
+ response = client.getNextNotification(latestID.getEventId()-1, 1, null);
+ SentryJSONAlterTableMessage alterTableMessage = deserializer.getAlterTableMessage(response.getEvents().get(0).getMessage());
+ assertEquals(HCatEventMessage.EventType.ALTER_TABLE, alterTableMessage.getEventType());
+ assertThat(alterTableMessage.getDB(), IsEqualIgnoringCase.equalToIgnoringCase(testDB));//dbName
+ assertThat(alterTableMessage.getTable(), IsEqualIgnoringCase.equalToIgnoringCase(testTable));//tableName
+ if(!useDbNotificationListener) {
+ Assert.assertEquals(oldLocation, alterTableMessage.getOldLocation()); //oldLocation
+ Assert.assertEquals(tbl1.getSd().getLocation(), alterTableMessage.getLocation()); //newLocation
+ }
+
+ //Alter table rename managed table - location also changes
+ // We need:
+ // - oldDbName
+ // - newDbName
+ // - oldTableName
+ // - newTableName
+ // - old location
+ // - new location
+ oldLocation = tbl1.getSd().getLocation();
+ String newDBName = testDB + random.nextInt(Integer.SIZE - 1);
+ String newTableName = testTable + random.nextInt(Integer.SIZE - 1);
+ String newLocation = tabDir1 + random.nextInt(Integer.SIZE - 1);
+ createMetastoreDB(client, newDBName);
+ previousID = latestID;
+ latestID = client.getCurrentNotificationEventId();
+ assertEquals(previousID.getEventId() + 1, latestID.getEventId());
+ alterTableRename(client, tbl1, newDBName, newTableName, newLocation);
+ previousID = latestID;
+ latestID = client.getCurrentNotificationEventId();
+ assertEquals(previousID.getEventId() + 1, latestID.getEventId());
+ response = client.getNextNotification(latestID.getEventId()-1, 1, null);
+ alterTableMessage = deserializer.getAlterTableMessage(response.getEvents().get(0).getMessage());
+ assertEquals(HCatEventMessage.EventType.ALTER_TABLE, alterTableMessage.getEventType());
+ assertThat(alterTableMessage.getDB(), IsEqualIgnoringCase.equalToIgnoringCase(testDB));//oldDbName
+ assertThat(alterTableMessage.getTable(), IsEqualIgnoringCase.equalToIgnoringCase(testTable));//oldTableName
+ assertThat(response.getEvents().get(0).getDbName(), IsEqualIgnoringCase.equalToIgnoringCase(newDBName));//newDbName
+ assertThat(response.getEvents().get(0).getTableName(), IsEqualIgnoringCase.equalToIgnoringCase(newTableName));//newTableName
+ if(!useDbNotificationListener) {
+ Assert.assertEquals(oldLocation, alterTableMessage.getOldLocation()); //oldLocation
+ Assert.assertEquals(tbl1.getSd().getLocation(), alterTableMessage.getLocation()); //newLocation
+ }
+ }
+
+ @Test
+ public void testAlterPartition() throws Exception {
+ testDB = "N_db" + random.nextInt(Integer.SIZE - 1);
+ String testTable = "N_table" + random.nextInt(Integer.SIZE - 1);
+
+ NotificationEventResponse response;
+ CurrentNotificationEventId latestID;
+ // Create database
+ createMetastoreDB(client, testDB);
+
+ // Create table with partition
+ Table tbl1 = createMetastoreTableWithPartition(client, testDB,
+ testTable, Lists.newArrayList(new FieldSchema("col1", "int", "")),
+ Lists.newArrayList(new FieldSchema("part_col1", "string", "")));
+ ArrayList<String> partVals1 = Lists.newArrayList("part1");
+ Partition partition = addPartition(client, testDB, testTable, partVals1, tbl1);
+
+ //Alter partition with location
+ // We need:
+ // - dbName
+ // - tableName
+ // - partition location
+ String oldLocation = tbl1.getSd().getLocation(); //TODO: SENTRY-1387: Tablelocation is stored instead of partition location
+ String newLocation = warehouseDir + File.separator + "newpart";
+ alterPartitionWithLocation(client, partition, newLocation);
+ latestID = client.getCurrentNotificationEventId();
+ response = client.getNextNotification(latestID.getEventId()-1, 1, null);
+ SentryJSONAlterPartitionMessage alterPartitionMessage = deserializer.getAlterPartitionMessage(response.getEvents().get(0).getMessage());
+ assertEquals(HCatEventMessage.EventType.ALTER_PARTITION, alterPartitionMessage.getEventType());
+ assertThat(alterPartitionMessage.getDB(), IsEqualIgnoringCase.equalToIgnoringCase(testDB));// dbName
+ assertThat(alterPartitionMessage.getTable(), IsEqualIgnoringCase.equalToIgnoringCase(testTable));// tableName
+ if(!useDbNotificationListener) {
+ Assert.assertEquals(oldLocation.toLowerCase(), alterPartitionMessage.getOldLocation());
+ Assert.assertEquals(newLocation.toLowerCase(), alterPartitionMessage.getLocation());
+ }
+ }
+}
+
[2/2] sentry git commit: SENTRY-1329: Adapt
SentryMetaStorePostEventListener to write HMS notification logs (Sravya
Tirukkovalur, Reviewed by: Hao Hao)
Posted by sr...@apache.org.
SENTRY-1329: Adapt SentryMetaStorePostEventListener to write HMS notification logs (Sravya Tirukkovalur, Reviewed by: Hao Hao)
Also,
1. Implementing the SentryJSONMessageFactory to add custom information in the notification log entry message, which includes
1.1. Implementing Message class for each message type
1.2. Implementing a deserializer
2. Implementing JSONAlterPartitionMessage and JSONAlterTableMessage to work around the issue in Hive 1.1.0. These classes do not have required default constructor.
3. Testing:
3.1. Sentry functionality: TestSentryListenerSentryDeserializer to verify functionality using Sentry's SentryMetastorePostEventListener and Sentry Notification log deserializer.
3.2. TestDbNotificationListenerSentryDeserializer uses Hive's DbNotificationListener and Sentry's JSON deserializeri. This would make sure Sentry is able to read the Notification logs written by Hive's DBNotificationListener
3.3. TestSentryListenerInBuiltDeserializer uses Sentry's SentryMetastorePostEventListener and Hive's inbuilt Notification log deserializer: This would make sure Sentry is not breaking other users of NotificationLog who might be using Hive's in built serializer
Change-Id: I680beb6db4e534bb0a9e6ee042ea0d4f33f0943f
Project: http://git-wip-us.apache.org/repos/asf/sentry/repo
Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/73d77bcd
Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/73d77bcd
Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/73d77bcd
Branch: refs/heads/sentry-ha-redesign
Commit: 73d77bcd0ce12ba5109a903fb51bf122df20c2db
Parents: a70cff9
Author: Sravya Tirukkovalur <sr...@apache.org>
Authored: Tue Jun 14 16:30:51 2016 -0700
Committer: Sravya Tirukkovalur <sr...@apache.org>
Committed: Wed Jul 13 13:45:07 2016 -0700
----------------------------------------------------------------------
sentry-binding/sentry-binding-hive/pom.xml | 6 +
.../SentryMetastorePostEventListener.java | 518 +++++++++----------
.../json/JSONAlterPartitionMessage.java | 78 +++
.../messaging/json/JSONAlterTableMessage.java | 68 +++
.../json/SentryJSONAddPartitionMessage.java | 49 ++
.../json/SentryJSONAlterPartitionMessage.java | 53 ++
.../json/SentryJSONAlterTableMessage.java | 50 ++
.../json/SentryJSONCreateDatabaseMessage.java | 44 ++
.../json/SentryJSONCreateTableMessage.java | 45 ++
.../json/SentryJSONDropDatabaseMessage.java | 44 ++
.../json/SentryJSONDropPartitionMessage.java | 49 ++
.../json/SentryJSONDropTableMessage.java | 45 ++
.../json/SentryJSONMessageDeserializer.java | 110 ++++
.../json/SentryJSONMessageFactory.java | 177 +++++++
.../TestDbPrivilegeCleanupOnDrop.java | 7 +-
.../AbstractTestWithStaticConfiguration.java | 3 +-
...actMetastoreTestWithStaticConfiguration.java | 5 +
...NotificationListenerInBuiltDeserializer.java | 353 +++++++++++++
...bNotificationListenerSentryDeserializer.java | 39 ++
...ificationLogUsingDBNotificationListener.java | 351 -------------
.../TestSentryListenerInBuiltDeserializer.java | 37 ++
.../TestSentryListenerSentryDeserializer.java | 375 ++++++++++++++
22 files changed, 1886 insertions(+), 620 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sentry/blob/73d77bcd/sentry-binding/sentry-binding-hive/pom.xml
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive/pom.xml b/sentry-binding/sentry-binding-hive/pom.xml
index 07aaae3..ca87836 100644
--- a/sentry-binding/sentry-binding-hive/pom.xml
+++ b/sentry-binding/sentry-binding-hive/pom.xml
@@ -106,6 +106,12 @@ limitations under the License.
<scope>provided</scope>
</dependency>
<dependency>
+ <groupId>org.apache.hive.hcatalog</groupId>
+ <artifactId>hive-hcatalog-server-extensions</artifactId>
+ <version>${hive.version}</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<scope>test</scope>
http://git-wip-us.apache.org/repos/asf/sentry/blob/73d77bcd/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentryMetastorePostEventListener.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentryMetastorePostEventListener.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentryMetastorePostEventListener.java
index d12ac15..75190c1 100644
--- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentryMetastorePostEventListener.java
+++ b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentryMetastorePostEventListener.java
@@ -17,15 +17,16 @@
*/
package org.apache.sentry.binding.metastore;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
+import java.util.concurrent.TimeUnit;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.MetaStoreEventListener;
+import org.apache.hadoop.hive.metastore.RawStore;
+import org.apache.hadoop.hive.metastore.RawStoreProxy;
+import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.NotificationEvent;
import org.apache.hadoop.hive.metastore.events.AddPartitionEvent;
import org.apache.hadoop.hive.metastore.events.AlterPartitionEvent;
import org.apache.hadoop.hive.metastore.events.AlterTableEvent;
@@ -34,371 +35,360 @@ import org.apache.hadoop.hive.metastore.events.CreateTableEvent;
import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent;
import org.apache.hadoop.hive.metastore.events.DropPartitionEvent;
import org.apache.hadoop.hive.metastore.events.DropTableEvent;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.sentry.core.common.exception.SentryUserException;
-import org.apache.sentry.binding.hive.conf.HiveAuthzConf;
-import org.apache.sentry.binding.hive.conf.HiveAuthzConf.AuthzConfVars;
-import org.apache.sentry.core.common.Authorizable;
-import org.apache.sentry.core.model.db.Database;
-import org.apache.sentry.core.model.db.Server;
-import org.apache.sentry.core.model.db.Table;
+import org.apache.hive.hcatalog.common.HCatConstants;
+import org.apache.sentry.binding.metastore.messaging.json.SentryJSONMessageFactory;
import org.apache.sentry.provider.db.SentryMetastoreListenerPlugin;
-import org.apache.sentry.provider.db.service.thrift.SentryPolicyServiceClient;
-import org.apache.sentry.service.thrift.SentryServiceClientFactory;
-import org.apache.sentry.service.thrift.ServiceConstants.ConfUtilties;
-import org.apache.sentry.service.thrift.ServiceConstants.ServerConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+/*
+A HMS listener class which should ideally go into the transaction which persists the Hive metadata.
+This class writes all DDL events to the NotificationLog through rawstore.addNotificationEvent(event)
+This class is very similar to DbNotificationListener, except:
+1. It uses a custom SentryJSONMessageFactory which adds additional information to the message part of the event
+ to avoid another round trip from the clients
+2. It handles the cases where actual operation has failed, and hence skips writing to the notification log.
+3. Has additional validations to make sure event has the required information.
+
+This can be replaced with DbNotificationListener in future and sentry's message factory can be plugged in if:
+- HIVE-14011 is fixed: Make MessageFactory truly pluggable
+- 2 and 3 above are handled in DbNotificationListener
+*/
+
public class SentryMetastorePostEventListener extends MetaStoreEventListener {
private static final Logger LOGGER = LoggerFactory.getLogger(SentryMetastoreListenerPlugin.class);
- private final HiveAuthzConf authzConf;
- private final Server server;
+ private RawStore rs;
+ private HiveConf hiveConf;
+ SentryJSONMessageFactory messageFactory;
- protected List<SentryMetastoreListenerPlugin> sentryPlugins = new ArrayList<SentryMetastoreListenerPlugin>();
+ private static SentryMetastorePostEventListener.CleanerThread cleaner = null;
- public SentryMetastorePostEventListener(Configuration config) {
- super(config);
-
- if (!(config instanceof HiveConf)) {
- String error = "Could not initialize Plugin - Configuration is not an instanceof HiveConf";
- LOGGER.error(error);
- throw new RuntimeException(error);
+ //Same as DbNotificationListener to make the transition back easy
+ private synchronized void init(HiveConf conf) {
+ try {
+ this.rs = RawStoreProxy.getProxy(conf, conf, conf.getVar(HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL), 999999);
+ } catch (MetaException var3) {
+ LOGGER.error("Unable to connect to raw store, notifications will not be tracked", var3);
+ this.rs = null;
}
- authzConf = HiveAuthzConf.getAuthzConf((HiveConf)config);
- server = new Server(authzConf.get(AuthzConfVars.AUTHZ_SERVER_NAME.getVar()));
- Iterable<String> pluginClasses = ConfUtilties.CLASS_SPLITTER
- .split(config.get(ServerConfig.SENTRY_METASTORE_PLUGINS,
- ServerConfig.SENTRY_METASTORE_PLUGINS_DEFAULT).trim());
+ if(cleaner == null && this.rs != null) {
+ cleaner = new SentryMetastorePostEventListener.CleanerThread(conf, this.rs);
+ cleaner.start();
+ }
+ }
- try {
- for (String pluginClassStr : pluginClasses) {
- Class<?> clazz = config.getClassByName(pluginClassStr);
- if (!SentryMetastoreListenerPlugin.class.isAssignableFrom(clazz)) {
- throw new IllegalArgumentException("Class ["
- + pluginClassStr + "] is not a "
- + SentryMetastoreListenerPlugin.class.getName());
- }
- SentryMetastoreListenerPlugin plugin = (SentryMetastoreListenerPlugin) clazz
- .getConstructor(Configuration.class, Configuration.class)
- .newInstance(config, authzConf);
- sentryPlugins.add(plugin);
- }
- } catch (Exception e) {
- LOGGER.error("Could not initialize Plugin !!", e);
- throw new RuntimeException(e);
+ public SentryMetastorePostEventListener(Configuration config) {
+ super(config);
+ // The code in MetastoreUtils.getMetaStoreListeners() that calls this looks for a constructor
+ // with a Configuration parameter, so we have to declare config as Configuration. But it
+ // actually passes a HiveConf, which we need. So we'll do this ugly down cast.
+ if (!(config instanceof HiveConf)) {
+ String error = "Could not initialize Plugin - Configuration is not an instanceof HiveConf";
+ LOGGER.error(error);
+ throw new RuntimeException(error);
}
+ hiveConf = (HiveConf)config;
+ messageFactory = new SentryJSONMessageFactory();
+ init(hiveConf);
}
@Override
- public void onCreateTable (CreateTableEvent tableEvent) throws MetaException {
+ public void onCreateDatabase(CreateDatabaseEvent dbEvent)
+ throws MetaException {
- // don't sync paths/privileges if the operation has failed
- if (!tableEvent.getStatus()) {
- LOGGER.debug("Skip sync paths/privileges with Sentry server for onCreateTable event," +
- " since the operation failed. \n");
+ // do not write to Notification log if the operation has failed
+ if (!dbEvent.getStatus()) {
+ LOGGER.info("Skipping writing to NotificationLog as the Create database event failed");
return;
}
- if (tableEvent.getTable().getSd().getLocation() != null) {
- String authzObj = tableEvent.getTable().getDbName() + "."
- + tableEvent.getTable().getTableName();
- String path = tableEvent.getTable().getSd().getLocation();
- for (SentryMetastoreListenerPlugin plugin : sentryPlugins) {
- plugin.addPath(authzObj, path);
- }
+ String location = dbEvent.getDatabase().getLocationUri();
+ if (location == null || location.isEmpty()) {
+ throw new SentryMalformedEventException("CreateDatabaseEvent has invalid location", dbEvent);
}
-
- // drop the privileges on the given table, in case if anything was left
- // behind during the drop
- if (!syncWithPolicyStore(AuthzConfVars.AUTHZ_SYNC_CREATE_WITH_POLICY_STORE)) {
- return;
+ String dbName = dbEvent.getDatabase().getName();
+ if (dbName == null || dbName.isEmpty()) {
+ throw new SentryMalformedEventException("CreateDatabaseEvent has invalid dbName", dbEvent);
}
- dropSentryTablePrivilege(tableEvent.getTable().getDbName(),
- tableEvent.getTable().getTableName());
+ NotificationEvent event = new NotificationEvent(0L, now(), HCatConstants.HCAT_CREATE_DATABASE_EVENT,
+ messageFactory.buildCreateDatabaseMessage(dbEvent.getDatabase()).toString());
+ event.setDbName(dbName);
+ this.enqueue(event);
}
@Override
- public void onDropTable(DropTableEvent tableEvent) throws MetaException {
-
- // don't sync paths/privileges if the operation has failed
- if (!tableEvent.getStatus()) {
- LOGGER.debug("Skip syncing paths/privileges with Sentry server for onDropTable event," +
- " since the operation failed. \n");
- return;
- }
+ public void onDropDatabase(DropDatabaseEvent dbEvent) throws MetaException {
- if (tableEvent.getTable().getSd().getLocation() != null) {
- String authzObj = tableEvent.getTable().getDbName() + "."
- + tableEvent.getTable().getTableName();
- for (SentryMetastoreListenerPlugin plugin : sentryPlugins) {
- plugin.removeAllPaths(authzObj, null);
- }
- }
- // drop the privileges on the given table
- if (!syncWithPolicyStore(AuthzConfVars.AUTHZ_SYNC_DROP_WITH_POLICY_STORE)) {
+ // do not write to Notification log if the operation has failed
+ if (!dbEvent.getStatus()) {
+ LOGGER.info("Skipping writing to NotificationLog as the Drop database event failed");
return;
}
- if (!tableEvent.getStatus()) {
- return;
+ String dbName = dbEvent.getDatabase().getName();
+ if (dbName == null || dbName.isEmpty()) {
+ throw new SentryMalformedEventException("DropDatabaseEvent has invalid dbName", dbEvent);
}
- dropSentryTablePrivilege(tableEvent.getTable().getDbName(),
- tableEvent.getTable().getTableName());
+ NotificationEvent event = new NotificationEvent(0L, now(), HCatConstants.HCAT_DROP_DATABASE_EVENT,
+ messageFactory.buildDropDatabaseMessage(dbEvent.getDatabase()).toString());
+ event.setDbName(dbName);
+ this.enqueue(event);
}
@Override
- public void onCreateDatabase(CreateDatabaseEvent dbEvent)
- throws MetaException {
+ public void onCreateTable (CreateTableEvent tableEvent) throws MetaException {
- // don't sync paths/privileges if the operation has failed
- if (!dbEvent.getStatus()) {
- LOGGER.debug("Skip syncing paths/privileges with Sentry server for onCreateDatabase event," +
- " since the operation failed. \n");
+ // do not write to Notification log if the operation has failed
+ if (!tableEvent.getStatus()) {
+ LOGGER.info("Skipping writing to NotificationLog as the Create table event failed");
return;
}
- if (dbEvent.getDatabase().getLocationUri() != null) {
- String authzObj = dbEvent.getDatabase().getName();
- String path = dbEvent.getDatabase().getLocationUri();
- for (SentryMetastoreListenerPlugin plugin : sentryPlugins) {
- plugin.addPath(authzObj, path);
- }
+ String dbName = tableEvent.getTable().getDbName();
+ if (dbName == null || dbName.isEmpty()) {
+ throw new SentryMalformedEventException("CreateTableEvent has invalid dbName", tableEvent);
}
- // drop the privileges on the database, in case anything left behind during
- // last drop db
- if (!syncWithPolicyStore(AuthzConfVars.AUTHZ_SYNC_CREATE_WITH_POLICY_STORE)) {
- return;
+ String tableName = tableEvent.getTable().getTableName();
+ if (tableName == null || tableName.isEmpty()) {
+ throw new SentryMalformedEventException("CreateTableEvent has invalid tableName", tableEvent);
}
-
- dropSentryDbPrivileges(dbEvent.getDatabase().getName());
+ // Create table event should also contain a location.
+ // But, Create view also generates a Create table event, but it does not have a location.
+ // Create view is identified by the tableType. But turns out tableType is not set in some cases.
+ // We assume that tableType is set for all create views.
+ //TODO: Location can be null/empty, handle that in HMSFollower
+ String tableType = tableEvent.getTable().getTableType();
+ if(!(tableType != null && tableType.equals(TableType.VIRTUAL_VIEW.name()))) {
+ if (tableType == null) {
+ LOGGER.warn("TableType is null, assuming it is not TableType.VIRTUAL_VIEW: tableEvent", tableEvent);
+ }
+ String location = tableEvent.getTable().getSd().getLocation();
+ if (location == null || location.isEmpty()) {
+ throw new SentryMalformedEventException("CreateTableEvent has invalid location", tableEvent);
+ }
+ }
+ NotificationEvent event = new NotificationEvent(0L, now(), HCatConstants.HCAT_CREATE_TABLE_EVENT,
+ messageFactory.buildCreateTableMessage(tableEvent.getTable()).toString());
+ event.setDbName(dbName);
+ event.setTableName(tableName);
+ this.enqueue(event);
}
- /**
- * Drop the privileges on the database. Note that child tables will be
- * dropped individually by client, so we just need to handle the removing
- * the db privileges. The table drop should cleanup the table privileges.
- */
@Override
- public void onDropDatabase(DropDatabaseEvent dbEvent) throws MetaException {
+ public void onDropTable(DropTableEvent tableEvent) throws MetaException {
- // don't sync paths/privileges if the operation has failed
- if (!dbEvent.getStatus()) {
- LOGGER.debug("Skip syncing paths/privileges with Sentry server for onDropDatabase event," +
- " since the operation failed. \n");
+ // do not write to Notification log if the operation has failed
+ if (!tableEvent.getStatus()) {
+ LOGGER.info("Skipping writing to NotificationLog as the Drop table event failed");
return;
}
- String authzObj = dbEvent.getDatabase().getName();
- for (SentryMetastoreListenerPlugin plugin : sentryPlugins) {
- List<String> tNames = dbEvent.getHandler().get_all_tables(authzObj);
- plugin.removeAllPaths(authzObj, tNames);
+ String dbName = tableEvent.getTable().getDbName();
+ if (dbName == null || dbName.isEmpty()) {
+ throw new SentryMalformedEventException("DropTableEvent has invalid dbName", tableEvent);
}
- if (!syncWithPolicyStore(AuthzConfVars.AUTHZ_SYNC_DROP_WITH_POLICY_STORE)) {
- return;
+ String tableName = tableEvent.getTable().getTableName();
+ if (tableName == null || tableName.isEmpty()) {
+ throw new SentryMalformedEventException("DropTableEvent has invalid tableName", tableEvent);
}
- dropSentryDbPrivileges(dbEvent.getDatabase().getName());
+ NotificationEvent event = new NotificationEvent(0L, now(), HCatConstants.HCAT_DROP_TABLE_EVENT,
+ messageFactory.buildDropTableMessage(tableEvent.getTable()).toString());
+ event.setDbName(dbName);
+ event.setTableName(tableName);
+ this.enqueue(event);
}
- /**
- * Adjust the privileges when table is renamed
- */
@Override
public void onAlterTable (AlterTableEvent tableEvent) throws MetaException {
- // don't sync privileges if the operation has failed
+ // do not write to Notification log if the operation has failed
if (!tableEvent.getStatus()) {
- LOGGER.debug("Skip syncing privileges with Sentry server for onAlterTable event," +
- " since the operation failed. \n");
+ LOGGER.info("Skipping writing to NotificationLog as the Alter table event failed");
return;
}
- renameSentryTablePrivilege(tableEvent.getOldTable().getDbName(),
- tableEvent.getOldTable().getTableName(),
- tableEvent.getOldTable().getSd().getLocation(),
- tableEvent.getNewTable().getDbName(),
- tableEvent.getNewTable().getTableName(),
- tableEvent.getNewTable().getSd().getLocation());
+ String dbName = tableEvent.getNewTable().getDbName();
+ if (dbName == null || dbName.isEmpty()) {
+ throw new SentryMalformedEventException("AlterTableEvent's newTable has invalid dbName", tableEvent);
+ }
+ String tableName = tableEvent.getNewTable().getTableName();
+ if (tableName == null || tableName.isEmpty()) {
+ throw new SentryMalformedEventException("AlterTableEvent's newTable has invalid tableName", tableEvent);
+ }
+ dbName = tableEvent.getOldTable().getDbName();
+ if (dbName == null || dbName.isEmpty()) {
+ throw new SentryMalformedEventException("AlterTableEvent's oldTable has invalid dbName", tableEvent);
+ }
+ tableName = tableEvent.getOldTable().getTableName();
+ if (tableName == null || tableName.isEmpty()) {
+ throw new SentryMalformedEventException("AlterTableEvent's oldTable has invalid tableName", tableEvent);
+ }
+ //Alter view also generates an alter table event, but it does not have a location
+ //TODO: Handle this case in Sentry
+ if(!tableEvent.getOldTable().getTableType().equals(TableType.VIRTUAL_VIEW.name())) {
+ String location = tableEvent.getNewTable().getSd().getLocation();
+ if (location == null || location.isEmpty()) {
+ throw new SentryMalformedEventException("AlterTableEvent's newTable has invalid location", tableEvent);
+ }
+ location = tableEvent.getOldTable().getSd().getLocation();
+ if (location == null || location.isEmpty()) {
+ throw new SentryMalformedEventException("AlterTableEvent's oldTable has invalid location", tableEvent);
+ }
+ }
+
+ NotificationEvent event = new NotificationEvent(0L, now(), HCatConstants.HCAT_ALTER_TABLE_EVENT,
+ messageFactory.buildAlterTableMessage(tableEvent.getOldTable(), tableEvent.getNewTable()).toString());
+ event.setDbName(tableEvent.getNewTable().getDbName());
+ event.setTableName(tableEvent.getNewTable().getTableName());
+ this.enqueue(event);
}
@Override
public void onAlterPartition(AlterPartitionEvent partitionEvent)
- throws MetaException {
+ throws MetaException {
- // don't sync privileges if the operation has failed
+ // do not write to Notification log if the operation has failed
if (!partitionEvent.getStatus()) {
- LOGGER.debug("Skip syncing privileges with Sentry server for onAlterPartition event," +
- " since the operation failed. \n");
+ LOGGER.info("Skipping writing to NotificationLog as the Alter partition event failed");
return;
}
- String oldLoc = null, newLoc = null;
- if (partitionEvent.getOldPartition() != null) {
- oldLoc = partitionEvent.getOldPartition().getSd().getLocation();
+ String dbName = partitionEvent.getNewPartition().getDbName();
+ if (dbName == null || dbName.isEmpty()) {
+ throw new SentryMalformedEventException("AlterPartitionEvent's newPartition has invalid dbName", partitionEvent);
}
- if (partitionEvent.getNewPartition() != null) {
- newLoc = partitionEvent.getNewPartition().getSd().getLocation();
+ String tableName = partitionEvent.getNewPartition().getTableName();
+ if (tableName == null || tableName.isEmpty()) {
+ throw new SentryMalformedEventException("AlterPartitionEvent's newPartition has invalid tableName", partitionEvent);
}
- if (oldLoc != null && newLoc != null && !oldLoc.equals(newLoc)) {
- String authzObj =
- partitionEvent.getOldPartition().getDbName() + "."
- + partitionEvent.getOldPartition().getTableName();
- for (SentryMetastoreListenerPlugin plugin : sentryPlugins) {
- plugin.renameAuthzObject(authzObj, oldLoc,
- authzObj, newLoc);
- }
- }
+ //TODO: Need more validations, but it is tricky as there are many variations and validations change for each one
+ // Alter partition Location
+ // Alter partition property
+ // Any more?
+
+ NotificationEvent event = new NotificationEvent(0L, now(), HCatConstants.HCAT_ALTER_PARTITION_EVENT,
+ messageFactory.buildAlterPartitionMessage(partitionEvent.getOldPartition(), partitionEvent.getNewPartition()).toString());
+
+ event.setDbName(partitionEvent.getNewPartition().getDbName());
+ event.setTableName(partitionEvent.getNewPartition().getTableName());
+ this.enqueue(event);
}
@Override
public void onAddPartition(AddPartitionEvent partitionEvent)
- throws MetaException {
+ throws MetaException {
- // don't sync path if the operation has failed
+ // do not write to Notification log if the operation has failed
if (!partitionEvent.getStatus()) {
- LOGGER.debug("Skip syncing path with Sentry server for onAddPartition event," +
- " since the operation failed. \n");
+ LOGGER.info("Skipping writing to NotificationLog as the Add partition event failed");
return;
}
- for (Partition part : partitionEvent.getPartitions()) {
- if (part.getSd() != null && part.getSd().getLocation() != null) {
- String authzObj = part.getDbName() + "." + part.getTableName();
- String path = part.getSd().getLocation();
- for (SentryMetastoreListenerPlugin plugin : sentryPlugins) {
- plugin.addPath(authzObj, path);
- }
- }
+ String dbName = partitionEvent.getTable().getDbName();
+ if (dbName == null || dbName.isEmpty()) {
+ throw new SentryMalformedEventException("AddPartitionEvent has invalid dbName", partitionEvent);
+ }
+ String tableName = partitionEvent.getTable().getTableName();
+ if (tableName == null || tableName.isEmpty()) {
+ throw new SentryMalformedEventException("AddPartitionEvent's newPartition has invalid tableName", partitionEvent);
}
- super.onAddPartition(partitionEvent);
+
+ //TODO: Need more validations?
+
+ NotificationEvent event = new NotificationEvent(0L, now(), HCatConstants.HCAT_ADD_PARTITION_EVENT,
+ messageFactory.buildAddPartitionMessage(partitionEvent.getTable(), partitionEvent.getPartitions()).toString());
+
+ event.setDbName(partitionEvent.getTable().getDbName());
+ event.setTableName(partitionEvent.getTable().getTableName());
+ this.enqueue(event);
}
@Override
public void onDropPartition(DropPartitionEvent partitionEvent)
- throws MetaException {
+ throws MetaException {
- // don't sync path if the operation has failed
+ // do not write to Notification log if the operation has failed
if (!partitionEvent.getStatus()) {
- LOGGER.debug("Skip syncing path with Sentry server for onDropPartition event," +
- " since the operation failed. \n");
+ LOGGER.info("Skipping writing to NotificationLog as the Drop partition event failed");
return;
}
- String authzObj = partitionEvent.getTable().getDbName() + "."
- + partitionEvent.getTable().getTableName();
- String path = partitionEvent.getPartition().getSd().getLocation();
- for (SentryMetastoreListenerPlugin plugin : sentryPlugins) {
- plugin.removePath(authzObj, path);
- }
- super.onDropPartition(partitionEvent);
+ NotificationEvent event = new NotificationEvent(0L, now(), HCatConstants.HCAT_DROP_PARTITION_EVENT,
+ messageFactory.buildDropPartitionMessage(partitionEvent.getTable(), partitionEvent.getPartition()).toString());
+ //TODO: Why is this asymmetric with add partitions(s)?
+ // Seems like adding multiple partitions generate a single event
+ // where as single partition drop generated an event?
+
+ event.setDbName(partitionEvent.getTable().getDbName());
+ event.setTableName(partitionEvent.getTable().getTableName());
+ this.enqueue(event);
}
- private SentryPolicyServiceClient getSentryServiceClient()
- throws MetaException {
- try {
- return SentryServiceClientFactory.create(authzConf);
- } catch (Exception e) {
- throw new MetaException("Failed to connect to Sentry service "
- + e.getMessage());
+ private int now() {
+ long millis = System.currentTimeMillis();
+ millis /= 1000;
+ if (millis > Integer.MAX_VALUE) {
+ LOGGER.warn("We've passed max int value in seconds since the epoch, " +
+ "all notification times will be the same!");
+ return Integer.MAX_VALUE;
}
+ return (int)millis;
}
- private void dropSentryDbPrivileges(String dbName) throws MetaException {
- List<Authorizable> authorizableTable = new ArrayList<Authorizable>();
- authorizableTable.add(server);
- authorizableTable.add(new Database(dbName));
- try {
- dropSentryPrivileges(authorizableTable);
- } catch (SentryUserException e) {
- throw new MetaException("Failed to remove Sentry policies for drop DB "
- + dbName + " Error: " + e.getMessage());
- } catch (IOException e) {
- throw new MetaException("Failed to find local user " + e.getMessage());
+ //Same as DbNotificationListener to make the transition back easy
+ private void enqueue(NotificationEvent event) {
+ if(this.rs != null) {
+ this.rs.addNotificationEvent(event);
+ } else {
+ LOGGER.warn("Dropping event " + event + " since notification is not running.");
}
-
}
- private void dropSentryTablePrivilege(String dbName, String tabName)
- throws MetaException {
- List<Authorizable> authorizableTable = new ArrayList<Authorizable>();
- authorizableTable.add(server);
- authorizableTable.add(new Database(dbName));
- authorizableTable.add(new Table(tabName));
+ //Same as DbNotificationListener to make the transition back easy
+ private static class CleanerThread extends Thread {
+ private RawStore rs;
+ private int ttl;
- try {
- dropSentryPrivileges(authorizableTable);
- } catch (SentryUserException e) {
- throw new MetaException(
- "Failed to remove Sentry policies for drop table " + dbName + "."
- + tabName + " Error: " + e.getMessage());
- } catch (IOException e) {
- throw new MetaException("Failed to find local user " + e.getMessage());
+ CleanerThread(HiveConf conf, RawStore rs) {
+ super("CleanerThread");
+ this.rs = rs;
+ this.setTimeToLive(conf.getTimeVar(HiveConf.ConfVars.METASTORE_EVENT_DB_LISTENER_TTL, TimeUnit.SECONDS));
+ this.setDaemon(true);
}
- }
- private void dropSentryPrivileges(
- List<? extends Authorizable> authorizableTable)
- throws SentryUserException, IOException, MetaException {
- String requestorUserName = UserGroupInformation.getCurrentUser()
- .getShortUserName();
- SentryPolicyServiceClient sentryClient = getSentryServiceClient();
- sentryClient.dropPrivileges(requestorUserName, authorizableTable);
-
- // Close the connection after dropping privileges is done.
- sentryClient.close();
- }
+ public void run() {
+ while(true) {
+ this.rs.cleanNotificationEvents(this.ttl);
- private void renameSentryTablePrivilege(String oldDbName, String oldTabName,
- String oldPath, String newDbName, String newTabName, String newPath)
- throws MetaException {
- List<Authorizable> oldAuthorizableTable = new ArrayList<Authorizable>();
- oldAuthorizableTable.add(server);
- oldAuthorizableTable.add(new Database(oldDbName));
- oldAuthorizableTable.add(new Table(oldTabName));
-
- List<Authorizable> newAuthorizableTable = new ArrayList<Authorizable>();
- newAuthorizableTable.add(server);
- newAuthorizableTable.add(new Database(newDbName));
- newAuthorizableTable.add(new Table(newTabName));
-
- if (!oldTabName.equalsIgnoreCase(newTabName)
- && syncWithPolicyStore(AuthzConfVars.AUTHZ_SYNC_ALTER_WITH_POLICY_STORE)) {
-
- SentryPolicyServiceClient sentryClient = getSentryServiceClient();
-
- try {
- String requestorUserName = UserGroupInformation.getCurrentUser()
- .getShortUserName();
- sentryClient.renamePrivileges(requestorUserName, oldAuthorizableTable, newAuthorizableTable);
- } catch (SentryUserException e) {
- throw new MetaException(
- "Failed to remove Sentry policies for rename table " + oldDbName
- + "." + oldTabName + "to " + newDbName + "." + newTabName
- + " Error: " + e.getMessage());
- } catch (IOException e) {
- throw new MetaException("Failed to find local user " + e.getMessage());
- } finally {
-
- // Close the connection after renaming privileges is done.
- sentryClient.close();
+ try {
+ Thread.sleep(60000L);
+ } catch (InterruptedException var2) {
+ LOGGER.info("Cleaner thread sleep interupted", var2);
+ }
}
}
- // The HDFS plugin needs to know if it's a path change (set location)
- for (SentryMetastoreListenerPlugin plugin : sentryPlugins) {
- plugin.renameAuthzObject(oldDbName + "." + oldTabName, oldPath,
- newDbName + "." + newTabName, newPath);
+
+ public void setTimeToLive(long configTtl) {
+ if(configTtl > 2147483647L) {
+ this.ttl = 2147483647;
+ } else {
+ this.ttl = (int)configTtl;
+ }
+
}
}
-
- private boolean syncWithPolicyStore(AuthzConfVars syncConfVar) {
- return "true"
- .equalsIgnoreCase(authzConf.get(syncConfVar.getVar(), "true"));
+ private class SentryMalformedEventException extends MetaException {
+ SentryMalformedEventException(String msg, Object event) {
+ //toString is not implemented in Event classes,
+ // hence using reflection to print the details of the Event object.
+ super(msg + "Event: " + ToStringBuilder.reflectionToString(event));
+ }
}
-
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/sentry/blob/73d77bcd/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterPartitionMessage.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterPartitionMessage.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterPartitionMessage.java
new file mode 100644
index 0000000..890186b
--- /dev/null
+++ b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterPartitionMessage.java
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.binding.metastore.messaging.json;
+
+import org.apache.hive.hcatalog.messaging.AlterPartitionMessage;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+import java.util.List;
+
+/*
+* This is only needed as corresponding class in Hive 1.1.0 does not have a default constructor
+ */
+public class JSONAlterPartitionMessage extends AlterPartitionMessage {
+ @JsonProperty
+ String server;
+ @JsonProperty
+ String servicePrincipal;
+ @JsonProperty
+ String db;
+ @JsonProperty
+ String table;
+ @JsonProperty
+ Long timestamp;
+ @JsonProperty
+ List<String> values;
+
+ public JSONAlterPartitionMessage() {}
+ public JSONAlterPartitionMessage(String server, String servicePrincipal, String db, String table, List<String> values, Long timestamp) {
+ this.server = server;
+ this.servicePrincipal = servicePrincipal;
+ this.db = db;
+ this.table = table;
+ this.timestamp = timestamp;
+ this.values = values;
+ this.checkValid();
+ }
+
+ public String getServer() {
+ return this.server;
+ }
+
+ public String getServicePrincipal() {
+ return this.servicePrincipal;
+ }
+
+ public String getDB() {
+ return this.db;
+ }
+
+ public Long getTimestamp() {
+ return this.timestamp;
+ }
+
+ public String getTable() {
+ return this.table;
+ }
+
+ public List<String> getValues() {
+ return this.values;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/sentry/blob/73d77bcd/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterTableMessage.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterTableMessage.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterTableMessage.java
new file mode 100644
index 0000000..76211c3
--- /dev/null
+++ b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterTableMessage.java
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.binding.metastore.messaging.json;
+
+import org.apache.hive.hcatalog.messaging.AlterTableMessage;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+/**
+ * This class is required as this class does not have a default contructor in Hive 1.1.0
+ */
+public class JSONAlterTableMessage extends AlterTableMessage {
+ @JsonProperty
+ String server;
+ @JsonProperty
+ String servicePrincipal;
+ @JsonProperty
+ String db;
+ @JsonProperty
+ String table;
+ @JsonProperty
+ Long timestamp;
+
+ public JSONAlterTableMessage() {}
+ public JSONAlterTableMessage(String server, String servicePrincipal, String db, String table, Long timestamp) {
+ this.server = server;
+ this.servicePrincipal = servicePrincipal;
+ this.db = db;
+ this.table = table;
+ this.timestamp = timestamp;
+ this.checkValid();
+ }
+
+ public String getServer() {
+ return this.server;
+ }
+
+ public String getServicePrincipal() {
+ return this.servicePrincipal;
+ }
+
+ public String getDB() {
+ return this.db;
+ }
+
+ public Long getTimestamp() {
+ return this.timestamp;
+ }
+
+ public String getTable() {
+ return this.table;
+ }
+}
http://git-wip-us.apache.org/repos/asf/sentry/blob/73d77bcd/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAddPartitionMessage.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAddPartitionMessage.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAddPartitionMessage.java
new file mode 100644
index 0000000..c0c469c
--- /dev/null
+++ b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAddPartitionMessage.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.binding.metastore.messaging.json;
+
+import org.apache.hive.hcatalog.messaging.json.JSONAddPartitionMessage;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+import java.util.List;
+import java.util.Map;
+
+public class SentryJSONAddPartitionMessage extends JSONAddPartitionMessage {
+ @JsonProperty
+ List<String> locations;
+
+ public SentryJSONAddPartitionMessage() {
+ }
+
+ public SentryJSONAddPartitionMessage(String server, String servicePrincipal, String db, String table,
+ List<Map<String, String>> partitions, Long timestamp, List<String> locations) {
+ super(server, servicePrincipal, db, table, partitions, timestamp);
+ this.locations = locations;
+ }
+
+ public List<String> getLocations() {
+ return locations;
+ }
+
+ @Override
+ public String toString() {
+ return SentryJSONMessageDeserializer.serialize(this);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/sentry/blob/73d77bcd/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterPartitionMessage.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterPartitionMessage.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterPartitionMessage.java
new file mode 100644
index 0000000..99eb67a
--- /dev/null
+++ b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterPartitionMessage.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.binding.metastore.messaging.json;
+
+import org.codehaus.jackson.annotate.JsonProperty;
+
+import java.util.List;
+
+public class SentryJSONAlterPartitionMessage extends JSONAlterPartitionMessage{
+ @JsonProperty
+ String location;
+ @JsonProperty
+ String oldLocation;
+
+ public SentryJSONAlterPartitionMessage() {
+ }
+
+ public SentryJSONAlterPartitionMessage(String server, String servicePrincipal, String db, String table,
+ List<String> values, Long timestamp, String oldlocation, String newLocation) {
+ super(server, servicePrincipal, db, table, values, timestamp);
+ this.location = newLocation;
+ this.oldLocation = oldlocation;
+ }
+
+ public String getLocation() {
+ return location;
+ }
+
+ public String getOldLocation() {
+ return oldLocation;
+ }
+
+ @Override
+ public String toString() {
+ return SentryJSONMessageDeserializer.serialize(this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/sentry/blob/73d77bcd/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterTableMessage.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterTableMessage.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterTableMessage.java
new file mode 100644
index 0000000..6e59e25
--- /dev/null
+++ b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterTableMessage.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.binding.metastore.messaging.json;
+
+import org.codehaus.jackson.annotate.JsonProperty;
+
+public class SentryJSONAlterTableMessage extends JSONAlterTableMessage {
+ @JsonProperty
+ String location; //newLocation
+ @JsonProperty
+ String oldLocation;
+
+ public SentryJSONAlterTableMessage() {
+ }
+
+ public SentryJSONAlterTableMessage(String server, String servicePrincipal, String db, String table,
+ Long timestamp, String oldLocation, String location) {
+ super(server, servicePrincipal, db, table, timestamp);
+ this.location = location;
+ this.oldLocation = oldLocation;
+ }
+
+ public String getLocation() {
+ return location;
+ }
+ public String getOldLocation() {
+ return oldLocation;
+ }
+
+ @Override
+ public String toString() {
+ return SentryJSONMessageDeserializer.serialize(this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/sentry/blob/73d77bcd/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateDatabaseMessage.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateDatabaseMessage.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateDatabaseMessage.java
new file mode 100644
index 0000000..ba19cbe
--- /dev/null
+++ b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateDatabaseMessage.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.binding.metastore.messaging.json;
+
+import org.apache.hive.hcatalog.messaging.json.JSONCreateDatabaseMessage;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+public class SentryJSONCreateDatabaseMessage extends JSONCreateDatabaseMessage {
+ @JsonProperty
+ String location;
+
+ public SentryJSONCreateDatabaseMessage() {
+ }
+
+ public SentryJSONCreateDatabaseMessage(String server, String servicePrincipal, String db, Long timestamp, String location) {
+ super(server, servicePrincipal, db, timestamp);
+ this.location = location;
+ }
+
+ public String getLocation() {
+ return location;
+ }
+
+ @Override
+ public String toString() {
+ return SentryJSONMessageDeserializer.serialize(this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/sentry/blob/73d77bcd/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateTableMessage.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateTableMessage.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateTableMessage.java
new file mode 100644
index 0000000..57d11d2
--- /dev/null
+++ b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateTableMessage.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.binding.metastore.messaging.json;
+
+import org.apache.hive.hcatalog.messaging.json.JSONCreateTableMessage;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+public class SentryJSONCreateTableMessage extends JSONCreateTableMessage {
+ @JsonProperty
+ String location;
+
+ public SentryJSONCreateTableMessage() {
+ }
+
+ public SentryJSONCreateTableMessage(String server, String servicePrincipal, String db, String table, Long timestamp, String location) {
+ super(server, servicePrincipal, db, table, timestamp);
+ this.location = location;
+ }
+
+ public String getLocation() {
+ return location;
+ }
+
+ @Override
+ public String toString() {
+ return SentryJSONMessageDeserializer.serialize(this);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/sentry/blob/73d77bcd/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropDatabaseMessage.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropDatabaseMessage.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropDatabaseMessage.java
new file mode 100644
index 0000000..05f83f7
--- /dev/null
+++ b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropDatabaseMessage.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.binding.metastore.messaging.json;
+
+import org.apache.hive.hcatalog.messaging.json.JSONDropDatabaseMessage;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+public class SentryJSONDropDatabaseMessage extends JSONDropDatabaseMessage{
+ @JsonProperty
+ String location;
+
+ public SentryJSONDropDatabaseMessage() {
+ }
+
+ public SentryJSONDropDatabaseMessage(String server, String servicePrincipal, String db, Long timestamp, String location) {
+ super(server, servicePrincipal, db, timestamp);
+ this.location = location;
+ }
+
+ public String getLocation() {
+ return location;
+ }
+
+ @Override
+ public String toString() {
+ return SentryJSONMessageDeserializer.serialize(this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/sentry/blob/73d77bcd/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropPartitionMessage.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropPartitionMessage.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropPartitionMessage.java
new file mode 100644
index 0000000..2ab61f7
--- /dev/null
+++ b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropPartitionMessage.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.binding.metastore.messaging.json;
+
+import org.apache.hive.hcatalog.messaging.json.JSONDropPartitionMessage;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+import java.util.List;
+import java.util.Map;
+
+public class SentryJSONDropPartitionMessage extends JSONDropPartitionMessage {
+ @JsonProperty
+ String location;
+
+ public SentryJSONDropPartitionMessage() {
+ }
+
+ public SentryJSONDropPartitionMessage(String server, String servicePrincipal, String db, String table,
+ List<Map<String, String>> partitions, Long timestamp, String location) {
+ super(server, servicePrincipal, db, table, partitions, timestamp);
+ this.location = location;
+ }
+
+ public String getLocation() {
+ return location;
+ }
+
+ @Override
+ public String toString() {
+ return SentryJSONMessageDeserializer.serialize(this);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/sentry/blob/73d77bcd/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropTableMessage.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropTableMessage.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropTableMessage.java
new file mode 100644
index 0000000..7005776
--- /dev/null
+++ b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropTableMessage.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.binding.metastore.messaging.json;
+
+import org.apache.hive.hcatalog.messaging.json.JSONDropTableMessage;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+
+public class SentryJSONDropTableMessage extends JSONDropTableMessage {
+ @JsonProperty
+ String location;
+
+ public SentryJSONDropTableMessage() {
+ }
+
+ public SentryJSONDropTableMessage(String server, String servicePrincipal, String db, String table, Long timestamp, String location) {
+ super(server, servicePrincipal, db, table, timestamp);
+ this.location = location;
+ }
+
+ public String getLocation() {
+ return location;
+ }
+
+ @Override
+ public String toString() {
+ return SentryJSONMessageDeserializer.serialize(this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/sentry/blob/73d77bcd/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageDeserializer.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageDeserializer.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageDeserializer.java
new file mode 100644
index 0000000..b645c45
--- /dev/null
+++ b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageDeserializer.java
@@ -0,0 +1,110 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.binding.metastore.messaging.json;
+
+import org.apache.hive.hcatalog.messaging.*;
+import org.codehaus.jackson.map.DeserializationConfig;
+import org.codehaus.jackson.map.ObjectMapper;
+
+public class SentryJSONMessageDeserializer extends MessageDeserializer {
+ static ObjectMapper mapper = new ObjectMapper();
+
+ public SentryJSONMessageDeserializer() {
+ }
+
+ /**
+ * Method to de-serialize CreateDatabaseMessage instance.
+ */
+ public SentryJSONCreateDatabaseMessage getCreateDatabaseMessage(String messageBody) {
+ try {
+ return (SentryJSONCreateDatabaseMessage)mapper.readValue(messageBody, SentryJSONCreateDatabaseMessage.class);
+ } catch (Exception var3) {
+ throw new IllegalArgumentException("Could not construct SentryJSONCreateDatabaseMessage.", var3);
+ }
+ }
+
+ public SentryJSONDropDatabaseMessage getDropDatabaseMessage(String messageBody) {
+ try {
+ return (SentryJSONDropDatabaseMessage)mapper.readValue(messageBody, SentryJSONDropDatabaseMessage.class);
+ } catch (Exception var3) {
+ throw new IllegalArgumentException("Could not construct SentryJSONDropDatabaseMessage.", var3);
+ }
+ }
+
+ public SentryJSONCreateTableMessage getCreateTableMessage(String messageBody) {
+ try {
+ return (SentryJSONCreateTableMessage)mapper.readValue(messageBody, SentryJSONCreateTableMessage.class);
+ } catch (Exception var3) {
+ throw new IllegalArgumentException("Could not construct SentryJSONCreateTableMessage.", var3);
+ }
+ }
+
+ public SentryJSONAlterTableMessage getAlterTableMessage(String messageBody) {
+ try {
+ return (SentryJSONAlterTableMessage)mapper.readValue(messageBody, SentryJSONAlterTableMessage.class);
+ } catch (Exception var3) {
+ throw new IllegalArgumentException("Could not construct SentryJSONAlterTableMessage.", var3);
+ }
+ }
+
+ public SentryJSONDropTableMessage getDropTableMessage(String messageBody) {
+ try {
+ return (SentryJSONDropTableMessage)mapper.readValue(messageBody, SentryJSONDropTableMessage.class);
+ } catch (Exception var3) {
+ throw new IllegalArgumentException("Could not construct SentryJSONDropTableMessage.", var3);
+ }
+ }
+
+ public SentryJSONAddPartitionMessage getAddPartitionMessage(String messageBody) {
+ try {
+ return (SentryJSONAddPartitionMessage)mapper.readValue(messageBody, SentryJSONAddPartitionMessage.class);
+ } catch (Exception var3) {
+ throw new IllegalArgumentException("Could not construct SentryJSONAddPartitionMessage.", var3);
+ }
+ }
+
+ public SentryJSONAlterPartitionMessage getAlterPartitionMessage(String messageBody) {
+ try {
+ return (SentryJSONAlterPartitionMessage)mapper.readValue(messageBody, SentryJSONAlterPartitionMessage.class);
+ } catch (Exception var3) {
+ throw new IllegalArgumentException("Could not construct SentryJSONAlterPartitionMessage.", var3);
+ }
+ }
+
+ public SentryJSONDropPartitionMessage getDropPartitionMessage(String messageBody) {
+ try {
+ return (SentryJSONDropPartitionMessage)mapper.readValue(messageBody, SentryJSONDropPartitionMessage.class);
+ } catch (Exception var3) {
+ throw new IllegalArgumentException("Could not construct SentryJSONDropPartitionMessage.", var3);
+ }
+ }
+
+ static {
+ mapper.configure(DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+ }
+
+ public static String serialize(Object object) {
+ try {
+ return mapper.writeValueAsString(object);
+ }
+ catch (Exception exception) {
+ throw new IllegalArgumentException("Could not serialize: ", exception);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/sentry/blob/73d77bcd/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageFactory.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageFactory.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageFactory.java
new file mode 100644
index 0000000..00e7db8
--- /dev/null
+++ b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageFactory.java
@@ -0,0 +1,177 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sentry.binding.metastore.messaging.json;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.common.classification.InterfaceAudience;
+import org.apache.hadoop.hive.common.classification.InterfaceStability;
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy;
+import org.apache.hive.hcatalog.messaging.*;
+
+import java.util.*;
+
+public class SentryJSONMessageFactory extends MessageFactory {
+ private static final Log LOG = LogFactory.getLog(SentryJSONMessageFactory.class.getName());
+ private static SentryJSONMessageDeserializer deserializer = new SentryJSONMessageDeserializer();
+ public SentryJSONMessageFactory() {
+ LOG.info("Using SentryJSONMessageFactory for building Notification log messages ");
+
+ }
+ public MessageDeserializer getDeserializer() {
+ return deserializer;
+ }
+
+ public String getVersion() {
+ return "0.1";
+ }
+
+ public String getMessageFormat() {
+ return "json";
+ }
+
+ public SentryJSONCreateDatabaseMessage buildCreateDatabaseMessage(Database db) {
+ return new SentryJSONCreateDatabaseMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, db.getName(),
+ Long.valueOf(this.now()), db.getLocationUri());
+ }
+ public SentryJSONDropDatabaseMessage buildDropDatabaseMessage(Database db) {
+ return new SentryJSONDropDatabaseMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, db.getName(),
+ Long.valueOf(this.now()), db.getLocationUri());
+ }
+
+ public SentryJSONCreateTableMessage buildCreateTableMessage(Table table) {
+ return new SentryJSONCreateTableMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, table.getDbName(),
+ table.getTableName(), Long.valueOf(this.now()), table.getSd().getLocation());
+ }
+
+ public SentryJSONAlterTableMessage buildAlterTableMessage(Table before, Table after) {
+ return new SentryJSONAlterTableMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, before.getDbName(),
+ before.getTableName(), Long.valueOf(this.now()), before.getSd().getLocation(), after.getSd().getLocation());
+ }
+
+ public SentryJSONDropTableMessage buildDropTableMessage(Table table) {
+ return new SentryJSONDropTableMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, table.getDbName(),
+ table.getTableName(), Long.valueOf(this.now()), table.getSd().getLocation());
+ }
+
+ public SentryJSONAddPartitionMessage buildAddPartitionMessage(Table table, List<Partition> partitions) {
+ return new SentryJSONAddPartitionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, table.getDbName(),
+ table.getTableName(), getPartitionKeyValues(table, partitions), Long.valueOf(this.now()),
+ getPartitionLocations(partitions));
+ }
+
+ private List<String> getPartitionLocations(List<Partition> partitions) {
+ List<String> paths = new ArrayList<String>();
+ for(Partition partition:partitions) {
+ paths.add(partition.getSd().getLocation());
+ }
+ return paths;
+ }
+
+ //TODO: Not sure what is this used for. Need to investigate
+ private List<String> getPartitionLocations(PartitionSpecProxy partitionSpec) {
+ Iterator<Partition> iterator = partitionSpec.getPartitionIterator();
+ List<String> locations = new ArrayList<String>();
+ while(iterator.hasNext()) {
+ locations.add(iterator.next().getSd().getLocation());
+ }
+ return locations;
+ }
+
+ @InterfaceAudience.LimitedPrivate({"Hive"})
+ @InterfaceStability.Evolving
+ public SentryJSONAddPartitionMessage buildAddPartitionMessage(Table table, PartitionSpecProxy partitionSpec) {
+ return new SentryJSONAddPartitionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, table.getDbName(),
+ table.getTableName(), getPartitionKeyValues(table, partitionSpec), Long.valueOf(this.now()),
+ getPartitionLocations(partitionSpec));
+ }
+
+ public SentryJSONAlterPartitionMessage buildAlterPartitionMessage(Partition before, Partition after) {
+ /*
+ f (partitionEvent.getOldPartition() != null) {
+ oldLoc = partitionEvent.getOldPartition().getSd().getLocation();
+ }
+ if (partitionEvent.getNewPartition() != null) {
+ newLoc = partitionEvent.getNewPartition().getSd().getLocation();
+ }
+
+ if ((oldLoc != null) && (newLoc != null) && (!oldLoc.equals(newLoc))) {
+ String authzObj =
+ partitionEvent.getOldPartition().getDbName() + "."
+ + partitionEvent.getOldPartition().getTableName();
+ for (SentryMetastoreListenerPlugin plugin : sentryPlugins) {
+ plugin.renameAuthzObject(authzObj, oldLoc,
+ authzObj, newLoc);
+ }
+ }
+ * */
+ return new SentryJSONAlterPartitionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, before.getDbName(),
+ before.getTableName(), before.getValues(), Long.valueOf(this.now()), before.getSd().getLocation(),
+ after.getSd().getLocation());
+ }
+
+ public SentryJSONDropPartitionMessage buildDropPartitionMessage(Table table, Partition partition) {
+ return new SentryJSONDropPartitionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, partition.getDbName(),
+ partition.getTableName(), Arrays.asList(getPartitionKeyValues(table, partition)),
+ Long.valueOf(this.now()), partition.getSd().getLocation());
+ }
+
+ private static Map<String, String> getPartitionKeyValues(Table table, Partition partition) {
+ LinkedHashMap partitionKeys = new LinkedHashMap();
+
+ for(int i = 0; i < table.getPartitionKeysSize(); ++i) {
+ partitionKeys.put(((FieldSchema)table.getPartitionKeys().get(i)).getName(), partition.getValues().get(i));
+ }
+
+ return partitionKeys;
+ }
+
+ private static List<Map<String, String>> getPartitionKeyValues(Table table, List<Partition> partitions) {
+ ArrayList partitionList = new ArrayList(partitions.size());
+ Iterator i$ = partitions.iterator();
+
+ while(i$.hasNext()) {
+ Partition partition = (Partition)i$.next();
+ partitionList.add(getPartitionKeyValues(table, partition));
+ }
+
+ return partitionList;
+ }
+
+ @InterfaceAudience.LimitedPrivate({"Hive"})
+ @InterfaceStability.Evolving
+ private static List<Map<String, String>> getPartitionKeyValues(Table table, PartitionSpecProxy partitionSpec) {
+ ArrayList partitionList = new ArrayList();
+ PartitionSpecProxy.PartitionIterator iterator = partitionSpec.getPartitionIterator();
+
+ while(iterator.hasNext()) {
+ Partition partition = (Partition)iterator.next();
+ partitionList.add(getPartitionKeyValues(table, partition));
+ }
+
+ return partitionList;
+ }
+ //This is private in parent class
+ private long now() {
+ return System.currentTimeMillis() / 1000L;
+ }
+}
http://git-wip-us.apache.org/repos/asf/sentry/blob/73d77bcd/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbPrivilegeCleanupOnDrop.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbPrivilegeCleanupOnDrop.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbPrivilegeCleanupOnDrop.java
index 767bcbe..439b9de 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbPrivilegeCleanupOnDrop.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbPrivilegeCleanupOnDrop.java
@@ -32,14 +32,13 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.sentry.tests.e2e.hive.AbstractTestWithStaticConfiguration;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
+import org.junit.*;
import com.google.common.collect.Lists;
import com.google.common.io.Resources;
+
+@Ignore("Ignoring until SENTRY-1321 is complete")
public class TestDbPrivilegeCleanupOnDrop extends
AbstractTestWithStaticConfiguration {
http://git-wip-us.apache.org/repos/asf/sentry/blob/73d77bcd/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java
index 2c4948e..7dc3d0f 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java
@@ -515,7 +515,8 @@ public abstract class AbstractTestWithStaticConfiguration {
} else {
properties.put(HiveConf.ConfVars.METASTORE_EVENT_LISTENERS.varname,
SentryMetastorePostEventListener.class.getName());
-
+ properties.put("hcatalog.message.factory.impl.json",
+ "org.apache.sentry.binding.metastore.messaging.json.SentryJSONMessageFactory");
}
}
}
http://git-wip-us.apache.org/repos/asf/sentry/blob/73d77bcd/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/AbstractMetastoreTestWithStaticConfiguration.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/AbstractMetastoreTestWithStaticConfiguration.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/AbstractMetastoreTestWithStaticConfiguration.java
index b72e317..567b4c8 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/AbstractMetastoreTestWithStaticConfiguration.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/AbstractMetastoreTestWithStaticConfiguration.java
@@ -200,6 +200,11 @@ public abstract class AbstractMetastoreTestWithStaticConfiguration extends
client.createDatabase(db);
}
+ public void dropMetastoreDBIfExists(HiveMetaStoreClient client, String dbName)
+ throws Exception {
+ client.dropDatabase(dbName, true, true, true);
+ }
+
public void execHiveSQLwithOverlay(final String sqlStmt,
final String userName, Map<String, String> overLay) throws Exception {
final HiveConf hiveConf = new HiveConf();