You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@sentry.apache.org by Sravya Tirukkovalur <sr...@apache.org> on 2016/07/12 20:12:34 UTC
Re: [2/4] sentry git commit: SENTRY-1329: Adapt SentryMetaStorePostEventListener
to write HMS notification logs
Sorry, pushed this change by mistake while pushing Sentry-1317. Fixed it
now.
Thanks!
On Tue, Jul 12, 2016 at 1:06 PM, <sr...@apache.org> wrote:
> SENTRY-1329: Adapt SentryMetaStorePostEventListener to write HMS
> notification logs
>
> Also,
> 1. Implementing the SentryJSONMessageFactory to add custom information in
> the notification log entry message, which includes
> 1.1. Implementing Message class for each message type
> 1.2. Implementing a deserializer
> 2. Implementing JSONAlterPartitionMessage and JSONAlterTableMessage to
> work around the issue in Hive 1.1.0. These classes do not have required
> default constructor.
> 3. Testing:
> 3.1. Sentry functionality: TestSentryListenerSentryDeserializer to verify
> functionality using Sentry's SentryMetastorePostEventListener and Sentry
> Notification log deserializer.
> 3.2. TestDbNotificationListenerSentryDeserializer uses Hive's
> DbNotificationListener and Sentry's JSON deserializeri. This would make
> sure Sentry is able to read the Notification logs written by Hive's
> DBNotificationListener
> 3.3. TestSentryListenerInBuiltDeserializer uses Sentry's
> SentryMetastorePostEventListener and Hive's inbuilt Notification log
> deserializer: This would make sure Sentry is not breaking other users of
> NotificationLog who might be using Hive's in built serializer
>
> Change-Id: I680beb6db4e534bb0a9e6ee042ea0d4f33f0943f
>
>
> Project: http://git-wip-us.apache.org/repos/asf/sentry/repo
> Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/113c5eae
> Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/113c5eae
> Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/113c5eae
>
> Branch: refs/heads/sentry-ha-redesign
> Commit: 113c5eae4ce9d3cb86ac7193731696d943fbea1f
> Parents: de7c26a
> Author: Sravya Tirukkovalur <sr...@apache.org>
> Authored: Tue Jun 14 16:30:51 2016 -0700
> Committer: Sravya Tirukkovalur <sr...@apache.org>
> Committed: Tue Jul 12 12:49:02 2016 -0700
>
> ----------------------------------------------------------------------
> sentry-binding/sentry-binding-hive/pom.xml | 6 +
> .../SentryMetastorePostEventListener.java | 518 +++++++++----------
> .../json/JSONAlterPartitionMessage.java | 78 +++
> .../messaging/json/JSONAlterTableMessage.java | 68 +++
> .../json/SentryJSONAddPartitionMessage.java | 49 ++
> .../json/SentryJSONAlterPartitionMessage.java | 53 ++
> .../json/SentryJSONAlterTableMessage.java | 50 ++
> .../json/SentryJSONCreateDatabaseMessage.java | 44 ++
> .../json/SentryJSONCreateTableMessage.java | 45 ++
> .../json/SentryJSONDropDatabaseMessage.java | 44 ++
> .../json/SentryJSONDropPartitionMessage.java | 49 ++
> .../json/SentryJSONDropTableMessage.java | 45 ++
> .../json/SentryJSONMessageDeserializer.java | 110 ++++
> .../json/SentryJSONMessageFactory.java | 177 +++++++
> .../TestDbPrivilegeCleanupOnDrop.java | 7 +-
> .../AbstractTestWithStaticConfiguration.java | 3 +-
> ...actMetastoreTestWithStaticConfiguration.java | 5 +
> ...NotificationListenerInBuiltDeserializer.java | 353 +++++++++++++
> ...bNotificationListenerSentryDeserializer.java | 39 ++
> ...ificationLogUsingDBNotificationListener.java | 351 -------------
> .../TestSentryListenerInBuiltDeserializer.java | 37 ++
> .../TestSentryListenerSentryDeserializer.java | 375 ++++++++++++++
> 22 files changed, 1886 insertions(+), 620 deletions(-)
> ----------------------------------------------------------------------
>
>
>
> http://git-wip-us.apache.org/repos/asf/sentry/blob/113c5eae/sentry-binding/sentry-binding-hive/pom.xml
> ----------------------------------------------------------------------
> diff --git a/sentry-binding/sentry-binding-hive/pom.xml
> b/sentry-binding/sentry-binding-hive/pom.xml
> index 07aaae3..ca87836 100644
> --- a/sentry-binding/sentry-binding-hive/pom.xml
> +++ b/sentry-binding/sentry-binding-hive/pom.xml
> @@ -106,6 +106,12 @@ limitations under the License.
> <scope>provided</scope>
> </dependency>
> <dependency>
> + <groupId>org.apache.hive.hcatalog</groupId>
> + <artifactId>hive-hcatalog-server-extensions</artifactId>
> + <version>${hive.version}</version>
> + <scope>compile</scope>
> + </dependency>
> + <dependency>
> <groupId>org.mockito</groupId>
> <artifactId>mockito-all</artifactId>
> <scope>test</scope>
>
>
> http://git-wip-us.apache.org/repos/asf/sentry/blob/113c5eae/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentryMetastorePostEventListener.java
> ----------------------------------------------------------------------
> diff --git
> a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentryMetastorePostEventListener.java
> b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentryMetastorePostEventListener.java
> index d12ac15..75190c1 100644
> ---
> a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentryMetastorePostEventListener.java
> +++
> b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentryMetastorePostEventListener.java
> @@ -17,15 +17,16 @@
> */
> package org.apache.sentry.binding.metastore;
>
> -import java.io.IOException;
> -import java.util.ArrayList;
> -import java.util.List;
> +import java.util.concurrent.TimeUnit;
>
> import org.apache.hadoop.conf.Configuration;
> import org.apache.hadoop.hive.conf.HiveConf;
> import org.apache.hadoop.hive.metastore.MetaStoreEventListener;
> +import org.apache.hadoop.hive.metastore.RawStore;
> +import org.apache.hadoop.hive.metastore.RawStoreProxy;
> +import org.apache.hadoop.hive.metastore.TableType;
> import org.apache.hadoop.hive.metastore.api.MetaException;
> -import org.apache.hadoop.hive.metastore.api.Partition;
> +import org.apache.hadoop.hive.metastore.api.NotificationEvent;
> import org.apache.hadoop.hive.metastore.events.AddPartitionEvent;
> import org.apache.hadoop.hive.metastore.events.AlterPartitionEvent;
> import org.apache.hadoop.hive.metastore.events.AlterTableEvent;
> @@ -34,371 +35,360 @@ import
> org.apache.hadoop.hive.metastore.events.CreateTableEvent;
> import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent;
> import org.apache.hadoop.hive.metastore.events.DropPartitionEvent;
> import org.apache.hadoop.hive.metastore.events.DropTableEvent;
> -import org.apache.hadoop.security.UserGroupInformation;
> -import org.apache.sentry.core.common.exception.SentryUserException;
> -import org.apache.sentry.binding.hive.conf.HiveAuthzConf;
> -import org.apache.sentry.binding.hive.conf.HiveAuthzConf.AuthzConfVars;
> -import org.apache.sentry.core.common.Authorizable;
> -import org.apache.sentry.core.model.db.Database;
> -import org.apache.sentry.core.model.db.Server;
> -import org.apache.sentry.core.model.db.Table;
> +import org.apache.hive.hcatalog.common.HCatConstants;
> +import
> org.apache.sentry.binding.metastore.messaging.json.SentryJSONMessageFactory;
> import org.apache.sentry.provider.db.SentryMetastoreListenerPlugin;
> -import
> org.apache.sentry.provider.db.service.thrift.SentryPolicyServiceClient;
> -import org.apache.sentry.service.thrift.SentryServiceClientFactory;
> -import org.apache.sentry.service.thrift.ServiceConstants.ConfUtilties;
> -import org.apache.sentry.service.thrift.ServiceConstants.ServerConfig;
> import org.slf4j.Logger;
> import org.slf4j.LoggerFactory;
>
> +import org.apache.commons.lang3.builder.ToStringBuilder;
> +/*
> +A HMS listener class which should ideally go into the transaction which
> persists the Hive metadata.
> +This class writes all DDL events to the NotificationLog through
> rawstore.addNotificationEvent(event)
> +This class is very similar to DbNotificationListener, except:
> +1. It uses a custom SentryJSONMessageFactory which adds additional
> information to the message part of the event
> + to avoid another round trip from the clients
> +2. It handles the cases where actual operation has failed, and hence
> skips writing to the notification log.
> +3. Has additional validations to make sure event has the required
> information.
> +
> +This can be replaced with DbNotificationListener in future and sentry's
> message factory can be plugged in if:
> +- HIVE-14011 is fixed: Make MessageFactory truly pluggable
> +- 2 and 3 above are handled in DbNotificationListener
> +*/
> +
> public class SentryMetastorePostEventListener extends
> MetaStoreEventListener {
>
> private static final Logger LOGGER =
> LoggerFactory.getLogger(SentryMetastoreListenerPlugin.class);
> - private final HiveAuthzConf authzConf;
> - private final Server server;
> + private RawStore rs;
> + private HiveConf hiveConf;
> + SentryJSONMessageFactory messageFactory;
>
> - protected List<SentryMetastoreListenerPlugin> sentryPlugins = new
> ArrayList<SentryMetastoreListenerPlugin>();
> + private static SentryMetastorePostEventListener.CleanerThread cleaner =
> null;
>
> - public SentryMetastorePostEventListener(Configuration config) {
> - super(config);
> -
> - if (!(config instanceof HiveConf)) {
> - String error = "Could not initialize Plugin - Configuration is
> not an instanceof HiveConf";
> - LOGGER.error(error);
> - throw new RuntimeException(error);
> + //Same as DbNotificationListener to make the transition back easy
> + private synchronized void init(HiveConf conf) {
> + try {
> + this.rs = RawStoreProxy.getProxy(conf, conf,
> conf.getVar(HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL), 999999);
> + } catch (MetaException var3) {
> + LOGGER.error("Unable to connect to raw store, notifications will
> not be tracked", var3);
> + this.rs = null;
> }
>
> - authzConf = HiveAuthzConf.getAuthzConf((HiveConf)config);
> - server = new
> Server(authzConf.get(AuthzConfVars.AUTHZ_SERVER_NAME.getVar()));
> - Iterable<String> pluginClasses = ConfUtilties.CLASS_SPLITTER
> - .split(config.get(ServerConfig.SENTRY_METASTORE_PLUGINS,
> - ServerConfig.SENTRY_METASTORE_PLUGINS_DEFAULT).trim());
> + if(cleaner == null && this.rs != null) {
> + cleaner = new SentryMetastorePostEventListener.CleanerThread(conf,
> this.rs);
> + cleaner.start();
> + }
> + }
>
> - try {
> - for (String pluginClassStr : pluginClasses) {
> - Class<?> clazz = config.getClassByName(pluginClassStr);
> - if (!SentryMetastoreListenerPlugin.class.isAssignableFrom(clazz))
> {
> - throw new IllegalArgumentException("Class ["
> - + pluginClassStr + "] is not a "
> - + SentryMetastoreListenerPlugin.class.getName());
> - }
> - SentryMetastoreListenerPlugin plugin =
> (SentryMetastoreListenerPlugin) clazz
> - .getConstructor(Configuration.class, Configuration.class)
> - .newInstance(config, authzConf);
> - sentryPlugins.add(plugin);
> - }
> - } catch (Exception e) {
> - LOGGER.error("Could not initialize Plugin !!", e);
> - throw new RuntimeException(e);
> + public SentryMetastorePostEventListener(Configuration config) {
> + super(config);
> + // The code in MetastoreUtils.getMetaStoreListeners() that calls this
> looks for a constructor
> + // with a Configuration parameter, so we have to declare config as
> Configuration. But it
> + // actually passes a HiveConf, which we need. So we'll do this ugly
> down cast.
> + if (!(config instanceof HiveConf)) {
> + String error = "Could not initialize Plugin - Configuration is not
> an instanceof HiveConf";
> + LOGGER.error(error);
> + throw new RuntimeException(error);
> }
> + hiveConf = (HiveConf)config;
> + messageFactory = new SentryJSONMessageFactory();
> + init(hiveConf);
> }
>
> @Override
> - public void onCreateTable (CreateTableEvent tableEvent) throws
> MetaException {
> + public void onCreateDatabase(CreateDatabaseEvent dbEvent)
> + throws MetaException {
>
> - // don't sync paths/privileges if the operation has failed
> - if (!tableEvent.getStatus()) {
> - LOGGER.debug("Skip sync paths/privileges with Sentry server for
> onCreateTable event," +
> - " since the operation failed. \n");
> + // do not write to Notification log if the operation has failed
> + if (!dbEvent.getStatus()) {
> + LOGGER.info("Skipping writing to NotificationLog as the Create
> database event failed");
> return;
> }
>
> - if (tableEvent.getTable().getSd().getLocation() != null) {
> - String authzObj = tableEvent.getTable().getDbName() + "."
> - + tableEvent.getTable().getTableName();
> - String path = tableEvent.getTable().getSd().getLocation();
> - for (SentryMetastoreListenerPlugin plugin : sentryPlugins) {
> - plugin.addPath(authzObj, path);
> - }
> + String location = dbEvent.getDatabase().getLocationUri();
> + if (location == null || location.isEmpty()) {
> + throw new SentryMalformedEventException("CreateDatabaseEvent has
> invalid location", dbEvent);
> }
> -
> - // drop the privileges on the given table, in case if anything was
> left
> - // behind during the drop
> - if
> (!syncWithPolicyStore(AuthzConfVars.AUTHZ_SYNC_CREATE_WITH_POLICY_STORE)) {
> - return;
> + String dbName = dbEvent.getDatabase().getName();
> + if (dbName == null || dbName.isEmpty()) {
> + throw new SentryMalformedEventException("CreateDatabaseEvent has
> invalid dbName", dbEvent);
> }
>
> - dropSentryTablePrivilege(tableEvent.getTable().getDbName(),
> - tableEvent.getTable().getTableName());
> + NotificationEvent event = new NotificationEvent(0L, now(),
> HCatConstants.HCAT_CREATE_DATABASE_EVENT,
> +
> messageFactory.buildCreateDatabaseMessage(dbEvent.getDatabase()).toString());
> + event.setDbName(dbName);
> + this.enqueue(event);
> }
>
> @Override
> - public void onDropTable(DropTableEvent tableEvent) throws MetaException
> {
> -
> - // don't sync paths/privileges if the operation has failed
> - if (!tableEvent.getStatus()) {
> - LOGGER.debug("Skip syncing paths/privileges with Sentry server for
> onDropTable event," +
> - " since the operation failed. \n");
> - return;
> - }
> + public void onDropDatabase(DropDatabaseEvent dbEvent) throws
> MetaException {
>
> - if (tableEvent.getTable().getSd().getLocation() != null) {
> - String authzObj = tableEvent.getTable().getDbName() + "."
> - + tableEvent.getTable().getTableName();
> - for (SentryMetastoreListenerPlugin plugin : sentryPlugins) {
> - plugin.removeAllPaths(authzObj, null);
> - }
> - }
> - // drop the privileges on the given table
> - if
> (!syncWithPolicyStore(AuthzConfVars.AUTHZ_SYNC_DROP_WITH_POLICY_STORE)) {
> + // do not write to Notification log if the operation has failed
> + if (!dbEvent.getStatus()) {
> + LOGGER.info("Skipping writing to NotificationLog as the Drop
> database event failed");
> return;
> }
>
> - if (!tableEvent.getStatus()) {
> - return;
> + String dbName = dbEvent.getDatabase().getName();
> + if (dbName == null || dbName.isEmpty()) {
> + throw new SentryMalformedEventException("DropDatabaseEvent has
> invalid dbName", dbEvent);
> }
>
> - dropSentryTablePrivilege(tableEvent.getTable().getDbName(),
> - tableEvent.getTable().getTableName());
> + NotificationEvent event = new NotificationEvent(0L, now(),
> HCatConstants.HCAT_DROP_DATABASE_EVENT,
> +
> messageFactory.buildDropDatabaseMessage(dbEvent.getDatabase()).toString());
> + event.setDbName(dbName);
> + this.enqueue(event);
> }
>
> @Override
> - public void onCreateDatabase(CreateDatabaseEvent dbEvent)
> - throws MetaException {
> + public void onCreateTable (CreateTableEvent tableEvent) throws
> MetaException {
>
> - // don't sync paths/privileges if the operation has failed
> - if (!dbEvent.getStatus()) {
> - LOGGER.debug("Skip syncing paths/privileges with Sentry server for
> onCreateDatabase event," +
> - " since the operation failed. \n");
> + // do not write to Notification log if the operation has failed
> + if (!tableEvent.getStatus()) {
> + LOGGER.info("Skipping writing to NotificationLog as the Create
> table event failed");
> return;
> }
>
> - if (dbEvent.getDatabase().getLocationUri() != null) {
> - String authzObj = dbEvent.getDatabase().getName();
> - String path = dbEvent.getDatabase().getLocationUri();
> - for (SentryMetastoreListenerPlugin plugin : sentryPlugins) {
> - plugin.addPath(authzObj, path);
> - }
> + String dbName = tableEvent.getTable().getDbName();
> + if (dbName == null || dbName.isEmpty()) {
> + throw new SentryMalformedEventException("CreateTableEvent has
> invalid dbName", tableEvent);
> }
> - // drop the privileges on the database, in case anything left behind
> during
> - // last drop db
> - if
> (!syncWithPolicyStore(AuthzConfVars.AUTHZ_SYNC_CREATE_WITH_POLICY_STORE)) {
> - return;
> + String tableName = tableEvent.getTable().getTableName();
> + if (tableName == null || tableName.isEmpty()) {
> + throw new SentryMalformedEventException("CreateTableEvent has
> invalid tableName", tableEvent);
> }
> -
> - dropSentryDbPrivileges(dbEvent.getDatabase().getName());
> + // Create table event should also contain a location.
> + // But, Create view also generates a Create table event, but it does
> not have a location.
> + // Create view is identified by the tableType. But turns out
> tableType is not set in some cases.
> + // We assume that tableType is set for all create views.
> + //TODO: Location can be null/empty, handle that in HMSFollower
> + String tableType = tableEvent.getTable().getTableType();
> + if(!(tableType != null && tableType.equals(
> TableType.VIRTUAL_VIEW.name()))) {
> + if (tableType == null) {
> + LOGGER.warn("TableType is null, assuming it is not
> TableType.VIRTUAL_VIEW: tableEvent", tableEvent);
> + }
> + String location = tableEvent.getTable().getSd().getLocation();
> + if (location == null || location.isEmpty()) {
> + throw new SentryMalformedEventException("CreateTableEvent has
> invalid location", tableEvent);
> + }
> + }
> + NotificationEvent event = new NotificationEvent(0L, now(),
> HCatConstants.HCAT_CREATE_TABLE_EVENT,
> +
> messageFactory.buildCreateTableMessage(tableEvent.getTable()).toString());
> + event.setDbName(dbName);
> + event.setTableName(tableName);
> + this.enqueue(event);
> }
>
> - /**
> - * Drop the privileges on the database. Note that child tables will be
> - * dropped individually by client, so we just need to handle the
> removing
> - * the db privileges. The table drop should cleanup the table
> privileges.
> - */
> @Override
> - public void onDropDatabase(DropDatabaseEvent dbEvent) throws
> MetaException {
> + public void onDropTable(DropTableEvent tableEvent) throws MetaException
> {
>
> - // don't sync paths/privileges if the operation has failed
> - if (!dbEvent.getStatus()) {
> - LOGGER.debug("Skip syncing paths/privileges with Sentry server for
> onDropDatabase event," +
> - " since the operation failed. \n");
> + // do not write to Notification log if the operation has failed
> + if (!tableEvent.getStatus()) {
> + LOGGER.info("Skipping writing to NotificationLog as the Drop table
> event failed");
> return;
> }
>
> - String authzObj = dbEvent.getDatabase().getName();
> - for (SentryMetastoreListenerPlugin plugin : sentryPlugins) {
> - List<String> tNames = dbEvent.getHandler().get_all_tables(authzObj);
> - plugin.removeAllPaths(authzObj, tNames);
> + String dbName = tableEvent.getTable().getDbName();
> + if (dbName == null || dbName.isEmpty()) {
> + throw new SentryMalformedEventException("DropTableEvent has invalid
> dbName", tableEvent);
> }
> - if
> (!syncWithPolicyStore(AuthzConfVars.AUTHZ_SYNC_DROP_WITH_POLICY_STORE)) {
> - return;
> + String tableName = tableEvent.getTable().getTableName();
> + if (tableName == null || tableName.isEmpty()) {
> + throw new SentryMalformedEventException("DropTableEvent has invalid
> tableName", tableEvent);
> }
>
> - dropSentryDbPrivileges(dbEvent.getDatabase().getName());
> + NotificationEvent event = new NotificationEvent(0L, now(),
> HCatConstants.HCAT_DROP_TABLE_EVENT,
> +
> messageFactory.buildDropTableMessage(tableEvent.getTable()).toString());
> + event.setDbName(dbName);
> + event.setTableName(tableName);
> + this.enqueue(event);
> }
>
> - /**
> - * Adjust the privileges when table is renamed
> - */
> @Override
> public void onAlterTable (AlterTableEvent tableEvent) throws
> MetaException {
>
> - // don't sync privileges if the operation has failed
> + // do not write to Notification log if the operation has failed
> if (!tableEvent.getStatus()) {
> - LOGGER.debug("Skip syncing privileges with Sentry server for
> onAlterTable event," +
> - " since the operation failed. \n");
> + LOGGER.info("Skipping writing to NotificationLog as the Alter table
> event failed");
> return;
> }
>
> - renameSentryTablePrivilege(tableEvent.getOldTable().getDbName(),
> - tableEvent.getOldTable().getTableName(),
> - tableEvent.getOldTable().getSd().getLocation(),
> - tableEvent.getNewTable().getDbName(),
> - tableEvent.getNewTable().getTableName(),
> - tableEvent.getNewTable().getSd().getLocation());
> + String dbName = tableEvent.getNewTable().getDbName();
> + if (dbName == null || dbName.isEmpty()) {
> + throw new SentryMalformedEventException("AlterTableEvent's newTable
> has invalid dbName", tableEvent);
> + }
> + String tableName = tableEvent.getNewTable().getTableName();
> + if (tableName == null || tableName.isEmpty()) {
> + throw new SentryMalformedEventException("AlterTableEvent's newTable
> has invalid tableName", tableEvent);
> + }
> + dbName = tableEvent.getOldTable().getDbName();
> + if (dbName == null || dbName.isEmpty()) {
> + throw new SentryMalformedEventException("AlterTableEvent's oldTable
> has invalid dbName", tableEvent);
> + }
> + tableName = tableEvent.getOldTable().getTableName();
> + if (tableName == null || tableName.isEmpty()) {
> + throw new SentryMalformedEventException("AlterTableEvent's oldTable
> has invalid tableName", tableEvent);
> + }
> + //Alter view also generates an alter table event, but it does not
> have a location
> + //TODO: Handle this case in Sentry
> + if(!tableEvent.getOldTable().getTableType().equals(
> TableType.VIRTUAL_VIEW.name())) {
> + String location = tableEvent.getNewTable().getSd().getLocation();
> + if (location == null || location.isEmpty()) {
> + throw new SentryMalformedEventException("AlterTableEvent's
> newTable has invalid location", tableEvent);
> + }
> + location = tableEvent.getOldTable().getSd().getLocation();
> + if (location == null || location.isEmpty()) {
> + throw new SentryMalformedEventException("AlterTableEvent's
> oldTable has invalid location", tableEvent);
> + }
> + }
> +
> + NotificationEvent event = new NotificationEvent(0L, now(),
> HCatConstants.HCAT_ALTER_TABLE_EVENT,
> +
> messageFactory.buildAlterTableMessage(tableEvent.getOldTable(),
> tableEvent.getNewTable()).toString());
> + event.setDbName(tableEvent.getNewTable().getDbName());
> + event.setTableName(tableEvent.getNewTable().getTableName());
> + this.enqueue(event);
> }
>
> @Override
> public void onAlterPartition(AlterPartitionEvent partitionEvent)
> - throws MetaException {
> + throws MetaException {
>
> - // don't sync privileges if the operation has failed
> + // do not write to Notification log if the operation has failed
> if (!partitionEvent.getStatus()) {
> - LOGGER.debug("Skip syncing privileges with Sentry server for
> onAlterPartition event," +
> - " since the operation failed. \n");
> + LOGGER.info("Skipping writing to NotificationLog as the Alter
> partition event failed");
> return;
> }
>
> - String oldLoc = null, newLoc = null;
> - if (partitionEvent.getOldPartition() != null) {
> - oldLoc = partitionEvent.getOldPartition().getSd().getLocation();
> + String dbName = partitionEvent.getNewPartition().getDbName();
> + if (dbName == null || dbName.isEmpty()) {
> + throw new SentryMalformedEventException("AlterPartitionEvent's
> newPartition has invalid dbName", partitionEvent);
> }
> - if (partitionEvent.getNewPartition() != null) {
> - newLoc = partitionEvent.getNewPartition().getSd().getLocation();
> + String tableName = partitionEvent.getNewPartition().getTableName();
> + if (tableName == null || tableName.isEmpty()) {
> + throw new SentryMalformedEventException("AlterPartitionEvent's
> newPartition has invalid tableName", partitionEvent);
> }
>
> - if (oldLoc != null && newLoc != null && !oldLoc.equals(newLoc)) {
> - String authzObj =
> - partitionEvent.getOldPartition().getDbName() + "."
> - + partitionEvent.getOldPartition().getTableName();
> - for (SentryMetastoreListenerPlugin plugin : sentryPlugins) {
> - plugin.renameAuthzObject(authzObj, oldLoc,
> - authzObj, newLoc);
> - }
> - }
> + //TODO: Need more validations, but it is tricky as there are many
> variations and validations change for each one
> + // Alter partition Location
> + // Alter partition property
> + // Any more?
> +
> + NotificationEvent event = new NotificationEvent(0L, now(),
> HCatConstants.HCAT_ALTER_PARTITION_EVENT,
> +
> messageFactory.buildAlterPartitionMessage(partitionEvent.getOldPartition(),
> partitionEvent.getNewPartition()).toString());
> +
> + event.setDbName(partitionEvent.getNewPartition().getDbName());
> + event.setTableName(partitionEvent.getNewPartition().getTableName());
> + this.enqueue(event);
> }
>
> @Override
> public void onAddPartition(AddPartitionEvent partitionEvent)
> - throws MetaException {
> + throws MetaException {
>
> - // don't sync path if the operation has failed
> + // do not write to Notification log if the operation has failed
> if (!partitionEvent.getStatus()) {
> - LOGGER.debug("Skip syncing path with Sentry server for
> onAddPartition event," +
> - " since the operation failed. \n");
> + LOGGER.info("Skipping writing to NotificationLog as the Add
> partition event failed");
> return;
> }
>
> - for (Partition part : partitionEvent.getPartitions()) {
> - if (part.getSd() != null && part.getSd().getLocation() != null) {
> - String authzObj = part.getDbName() + "." + part.getTableName();
> - String path = part.getSd().getLocation();
> - for (SentryMetastoreListenerPlugin plugin : sentryPlugins) {
> - plugin.addPath(authzObj, path);
> - }
> - }
> + String dbName = partitionEvent.getTable().getDbName();
> + if (dbName == null || dbName.isEmpty()) {
> + throw new SentryMalformedEventException("AddPartitionEvent has
> invalid dbName", partitionEvent);
> + }
> + String tableName = partitionEvent.getTable().getTableName();
> + if (tableName == null || tableName.isEmpty()) {
> + throw new SentryMalformedEventException("AddPartitionEvent's
> newPartition has invalid tableName", partitionEvent);
> }
> - super.onAddPartition(partitionEvent);
> +
> + //TODO: Need more validations?
> +
> + NotificationEvent event = new NotificationEvent(0L, now(),
> HCatConstants.HCAT_ADD_PARTITION_EVENT,
> +
> messageFactory.buildAddPartitionMessage(partitionEvent.getTable(),
> partitionEvent.getPartitions()).toString());
> +
> + event.setDbName(partitionEvent.getTable().getDbName());
> + event.setTableName(partitionEvent.getTable().getTableName());
> + this.enqueue(event);
> }
>
> @Override
> public void onDropPartition(DropPartitionEvent partitionEvent)
> - throws MetaException {
> + throws MetaException {
>
> - // don't sync path if the operation has failed
> + // do not write to Notification log if the operation has failed
> if (!partitionEvent.getStatus()) {
> - LOGGER.debug("Skip syncing path with Sentry server for
> onDropPartition event," +
> - " since the operation failed. \n");
> + LOGGER.info("Skipping writing to NotificationLog as the Drop
> partition event failed");
> return;
> }
>
> - String authzObj = partitionEvent.getTable().getDbName() + "."
> - + partitionEvent.getTable().getTableName();
> - String path = partitionEvent.getPartition().getSd().getLocation();
> - for (SentryMetastoreListenerPlugin plugin : sentryPlugins) {
> - plugin.removePath(authzObj, path);
> - }
> - super.onDropPartition(partitionEvent);
> + NotificationEvent event = new NotificationEvent(0L, now(),
> HCatConstants.HCAT_DROP_PARTITION_EVENT,
> +
> messageFactory.buildDropPartitionMessage(partitionEvent.getTable(),
> partitionEvent.getPartition()).toString());
> + //TODO: Why is this asymmetric with add partitions(s)?
> + // Seems like adding multiple partitions generate a single event
> + // where as single partition drop generated an event?
> +
> + event.setDbName(partitionEvent.getTable().getDbName());
> + event.setTableName(partitionEvent.getTable().getTableName());
> + this.enqueue(event);
> }
>
> - private SentryPolicyServiceClient getSentryServiceClient()
> - throws MetaException {
> - try {
> - return SentryServiceClientFactory.create(authzConf);
> - } catch (Exception e) {
> - throw new MetaException("Failed to connect to Sentry service "
> - + e.getMessage());
> + private int now() {
> + long millis = System.currentTimeMillis();
> + millis /= 1000;
> + if (millis > Integer.MAX_VALUE) {
> + LOGGER.warn("We've passed max int value in seconds since the epoch,
> " +
> + "all notification times will be the same!");
> + return Integer.MAX_VALUE;
> }
> + return (int)millis;
> }
>
> - private void dropSentryDbPrivileges(String dbName) throws MetaException
> {
> - List<Authorizable> authorizableTable = new ArrayList<Authorizable>();
> - authorizableTable.add(server);
> - authorizableTable.add(new Database(dbName));
> - try {
> - dropSentryPrivileges(authorizableTable);
> - } catch (SentryUserException e) {
> - throw new MetaException("Failed to remove Sentry policies for drop
> DB "
> - + dbName + " Error: " + e.getMessage());
> - } catch (IOException e) {
> - throw new MetaException("Failed to find local user " +
> e.getMessage());
> + //Same as DbNotificationListener to make the transition back easy
> + private void enqueue(NotificationEvent event) {
> + if(this.rs != null) {
> + this.rs.addNotificationEvent(event);
> + } else {
> + LOGGER.warn("Dropping event " + event + " since notification is not
> running.");
> }
> -
> }
>
> - private void dropSentryTablePrivilege(String dbName, String tabName)
> - throws MetaException {
> - List<Authorizable> authorizableTable = new ArrayList<Authorizable>();
> - authorizableTable.add(server);
> - authorizableTable.add(new Database(dbName));
> - authorizableTable.add(new Table(tabName));
> + //Same as DbNotificationListener to make the transition back easy
> + private static class CleanerThread extends Thread {
> + private RawStore rs;
> + private int ttl;
>
> - try {
> - dropSentryPrivileges(authorizableTable);
> - } catch (SentryUserException e) {
> - throw new MetaException(
> - "Failed to remove Sentry policies for drop table " + dbName +
> "."
> - + tabName + " Error: " + e.getMessage());
> - } catch (IOException e) {
> - throw new MetaException("Failed to find local user " +
> e.getMessage());
> + CleanerThread(HiveConf conf, RawStore rs) {
> + super("CleanerThread");
> + this.rs = rs;
> +
> this.setTimeToLive(conf.getTimeVar(HiveConf.ConfVars.METASTORE_EVENT_DB_LISTENER_TTL,
> TimeUnit.SECONDS));
> + this.setDaemon(true);
> }
>
> - }
> - private void dropSentryPrivileges(
> - List<? extends Authorizable> authorizableTable)
> - throws SentryUserException, IOException, MetaException {
> - String requestorUserName = UserGroupInformation.getCurrentUser()
> - .getShortUserName();
> - SentryPolicyServiceClient sentryClient = getSentryServiceClient();
> - sentryClient.dropPrivileges(requestorUserName, authorizableTable);
> -
> - // Close the connection after dropping privileges is done.
> - sentryClient.close();
> - }
> + public void run() {
> + while(true) {
> + this.rs.cleanNotificationEvents(this.ttl);
>
> - private void renameSentryTablePrivilege(String oldDbName, String
> oldTabName,
> - String oldPath, String newDbName, String newTabName, String newPath)
> - throws MetaException {
> - List<Authorizable> oldAuthorizableTable = new
> ArrayList<Authorizable>();
> - oldAuthorizableTable.add(server);
> - oldAuthorizableTable.add(new Database(oldDbName));
> - oldAuthorizableTable.add(new Table(oldTabName));
> -
> - List<Authorizable> newAuthorizableTable = new
> ArrayList<Authorizable>();
> - newAuthorizableTable.add(server);
> - newAuthorizableTable.add(new Database(newDbName));
> - newAuthorizableTable.add(new Table(newTabName));
> -
> - if (!oldTabName.equalsIgnoreCase(newTabName)
> - &&
> syncWithPolicyStore(AuthzConfVars.AUTHZ_SYNC_ALTER_WITH_POLICY_STORE)) {
> -
> - SentryPolicyServiceClient sentryClient = getSentryServiceClient();
> -
> - try {
> - String requestorUserName = UserGroupInformation.getCurrentUser()
> - .getShortUserName();
> - sentryClient.renamePrivileges(requestorUserName,
> oldAuthorizableTable, newAuthorizableTable);
> - } catch (SentryUserException e) {
> - throw new MetaException(
> - "Failed to remove Sentry policies for rename table " +
> oldDbName
> - + "." + oldTabName + "to " + newDbName + "." + newTabName
> - + " Error: " + e.getMessage());
> - } catch (IOException e) {
> - throw new MetaException("Failed to find local user " +
> e.getMessage());
> - } finally {
> -
> - // Close the connection after renaming privileges is done.
> - sentryClient.close();
> + try {
> + Thread.sleep(60000L);
> + } catch (InterruptedException var2) {
> + LOGGER.info("Cleaner thread sleep interupted", var2);
> + }
> }
> }
> - // The HDFS plugin needs to know if it's a path change (set location)
> - for (SentryMetastoreListenerPlugin plugin : sentryPlugins) {
> - plugin.renameAuthzObject(oldDbName + "." + oldTabName, oldPath,
> - newDbName + "." + newTabName, newPath);
> +
> + public void setTimeToLive(long configTtl) {
> + if(configTtl > 2147483647L) {
> + this.ttl = 2147483647;
> + } else {
> + this.ttl = (int)configTtl;
> + }
> +
> }
> }
> -
> - private boolean syncWithPolicyStore(AuthzConfVars syncConfVar) {
> - return "true"
> - .equalsIgnoreCase(authzConf.get(syncConfVar.getVar(), "true"));
> + private class SentryMalformedEventException extends MetaException {
> + SentryMalformedEventException(String msg, Object event) {
> + //toString is not implemented in Event classes,
> + // hence using reflection to print the details of the Event object.
> + super(msg + "Event: " + ToStringBuilder.reflectionToString(event));
> + }
> }
> -
> -}
> +}
> \ No newline at end of file
>
>
> http://git-wip-us.apache.org/repos/asf/sentry/blob/113c5eae/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterPartitionMessage.java
> ----------------------------------------------------------------------
> diff --git
> a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterPartitionMessage.java
> b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterPartitionMessage.java
> new file mode 100644
> index 0000000..890186b
> --- /dev/null
> +++
> b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterPartitionMessage.java
> @@ -0,0 +1,78 @@
> +/**
> + * Licensed to the Apache Software Foundation (ASF) under one
> + * or more contributor license agreements. See the NOTICE file
> + * distributed with this work for additional information
> + * regarding copyright ownership. The ASF licenses this file
> + * to you under the Apache License, Version 2.0 (the
> + * "License"); you may not use this file except in compliance
> + * with the License. You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> + */
> +
> +package org.apache.sentry.binding.metastore.messaging.json;
> +
> +import org.apache.hive.hcatalog.messaging.AlterPartitionMessage;
> +import org.codehaus.jackson.annotate.JsonProperty;
> +
> +import java.util.List;
> +
> +/*
> +* This is only needed as corresponding class in Hive 1.1.0 does not have
> a default constructor
> + */
> +public class JSONAlterPartitionMessage extends AlterPartitionMessage {
> + @JsonProperty
> + String server;
> + @JsonProperty
> + String servicePrincipal;
> + @JsonProperty
> + String db;
> + @JsonProperty
> + String table;
> + @JsonProperty
> + Long timestamp;
> + @JsonProperty
> + List<String> values;
> +
> + public JSONAlterPartitionMessage() {}
> + public JSONAlterPartitionMessage(String server, String
> servicePrincipal, String db, String table, List<String> values, Long
> timestamp) {
> + this.server = server;
> + this.servicePrincipal = servicePrincipal;
> + this.db = db;
> + this.table = table;
> + this.timestamp = timestamp;
> + this.values = values;
> + this.checkValid();
> + }
> +
> + public String getServer() {
> + return this.server;
> + }
> +
> + public String getServicePrincipal() {
> + return this.servicePrincipal;
> + }
> +
> + public String getDB() {
> + return this.db;
> + }
> +
> + public Long getTimestamp() {
> + return this.timestamp;
> + }
> +
> + public String getTable() {
> + return this.table;
> + }
> +
> + public List<String> getValues() {
> + return this.values;
> + }
> +
> +}
> \ No newline at end of file
>
>
> http://git-wip-us.apache.org/repos/asf/sentry/blob/113c5eae/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterTableMessage.java
> ----------------------------------------------------------------------
> diff --git
> a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterTableMessage.java
> b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterTableMessage.java
> new file mode 100644
> index 0000000..76211c3
> --- /dev/null
> +++
> b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterTableMessage.java
> @@ -0,0 +1,68 @@
> +/**
> + * Licensed to the Apache Software Foundation (ASF) under one
> + * or more contributor license agreements. See the NOTICE file
> + * distributed with this work for additional information
> + * regarding copyright ownership. The ASF licenses this file
> + * to you under the Apache License, Version 2.0 (the
> + * "License"); you may not use this file except in compliance
> + * with the License. You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> + */
> +
> +package org.apache.sentry.binding.metastore.messaging.json;
> +
> +import org.apache.hive.hcatalog.messaging.AlterTableMessage;
> +import org.codehaus.jackson.annotate.JsonProperty;
> +
> +/**
> + * This class is required as this class does not have a default
> contructor in Hive 1.1.0
> + */
> +public class JSONAlterTableMessage extends AlterTableMessage {
> + @JsonProperty
> + String server;
> + @JsonProperty
> + String servicePrincipal;
> + @JsonProperty
> + String db;
> + @JsonProperty
> + String table;
> + @JsonProperty
> + Long timestamp;
> +
> + public JSONAlterTableMessage() {}
> + public JSONAlterTableMessage(String server, String servicePrincipal,
> String db, String table, Long timestamp) {
> + this.server = server;
> + this.servicePrincipal = servicePrincipal;
> + this.db = db;
> + this.table = table;
> + this.timestamp = timestamp;
> + this.checkValid();
> + }
> +
> + public String getServer() {
> + return this.server;
> + }
> +
> + public String getServicePrincipal() {
> + return this.servicePrincipal;
> + }
> +
> + public String getDB() {
> + return this.db;
> + }
> +
> + public Long getTimestamp() {
> + return this.timestamp;
> + }
> +
> + public String getTable() {
> + return this.table;
> + }
> +}
>
>
> http://git-wip-us.apache.org/repos/asf/sentry/blob/113c5eae/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAddPartitionMessage.java
> ----------------------------------------------------------------------
> diff --git
> a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAddPartitionMessage.java
> b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAddPartitionMessage.java
> new file mode 100644
> index 0000000..c0c469c
> --- /dev/null
> +++
> b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAddPartitionMessage.java
> @@ -0,0 +1,49 @@
> +/**
> + * Licensed to the Apache Software Foundation (ASF) under one
> + * or more contributor license agreements. See the NOTICE file
> + * distributed with this work for additional information
> + * regarding copyright ownership. The ASF licenses this file
> + * to you under the Apache License, Version 2.0 (the
> + * "License"); you may not use this file except in compliance
> + * with the License. You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> + */
> +
> +package org.apache.sentry.binding.metastore.messaging.json;
> +
> +import org.apache.hive.hcatalog.messaging.json.JSONAddPartitionMessage;
> +import org.codehaus.jackson.annotate.JsonProperty;
> +
> +import java.util.List;
> +import java.util.Map;
> +
> +public class SentryJSONAddPartitionMessage extends
> JSONAddPartitionMessage {
> + @JsonProperty
> + List<String> locations;
> +
> + public SentryJSONAddPartitionMessage() {
> + }
> +
> + public SentryJSONAddPartitionMessage(String server, String
> servicePrincipal, String db, String table,
> + List<Map<String, String>>
> partitions, Long timestamp, List<String> locations) {
> + super(server, servicePrincipal, db, table, partitions, timestamp);
> + this.locations = locations;
> + }
> +
> + public List<String> getLocations() {
> + return locations;
> + }
> +
> + @Override
> + public String toString() {
> + return SentryJSONMessageDeserializer.serialize(this);
> + }
> +
> +}
>
>
> http://git-wip-us.apache.org/repos/asf/sentry/blob/113c5eae/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterPartitionMessage.java
> ----------------------------------------------------------------------
> diff --git
> a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterPartitionMessage.java
> b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterPartitionMessage.java
> new file mode 100644
> index 0000000..99eb67a
> --- /dev/null
> +++
> b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterPartitionMessage.java
> @@ -0,0 +1,53 @@
> +/**
> + * Licensed to the Apache Software Foundation (ASF) under one
> + * or more contributor license agreements. See the NOTICE file
> + * distributed with this work for additional information
> + * regarding copyright ownership. The ASF licenses this file
> + * to you under the Apache License, Version 2.0 (the
> + * "License"); you may not use this file except in compliance
> + * with the License. You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> + */
> +
> +package org.apache.sentry.binding.metastore.messaging.json;
> +
> +import org.codehaus.jackson.annotate.JsonProperty;
> +
> +import java.util.List;
> +
> +public class SentryJSONAlterPartitionMessage extends
> JSONAlterPartitionMessage{
> + @JsonProperty
> + String location;
> + @JsonProperty
> + String oldLocation;
> +
> + public SentryJSONAlterPartitionMessage() {
> + }
> +
> + public SentryJSONAlterPartitionMessage(String server, String
> servicePrincipal, String db, String table,
> + List<String> values, Long
> timestamp, String oldlocation, String newLocation) {
> + super(server, servicePrincipal, db, table, values, timestamp);
> + this.location = newLocation;
> + this.oldLocation = oldlocation;
> + }
> +
> + public String getLocation() {
> + return location;
> + }
> +
> + public String getOldLocation() {
> + return oldLocation;
> + }
> +
> + @Override
> + public String toString() {
> + return SentryJSONMessageDeserializer.serialize(this);
> + }
> +}
>
>
> http://git-wip-us.apache.org/repos/asf/sentry/blob/113c5eae/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterTableMessage.java
> ----------------------------------------------------------------------
> diff --git
> a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterTableMessage.java
> b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterTableMessage.java
> new file mode 100644
> index 0000000..6e59e25
> --- /dev/null
> +++
> b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterTableMessage.java
> @@ -0,0 +1,50 @@
> +/**
> + * Licensed to the Apache Software Foundation (ASF) under one
> + * or more contributor license agreements. See the NOTICE file
> + * distributed with this work for additional information
> + * regarding copyright ownership. The ASF licenses this file
> + * to you under the Apache License, Version 2.0 (the
> + * "License"); you may not use this file except in compliance
> + * with the License. You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> + */
> +
> +package org.apache.sentry.binding.metastore.messaging.json;
> +
> +import org.codehaus.jackson.annotate.JsonProperty;
> +
> +public class SentryJSONAlterTableMessage extends JSONAlterTableMessage {
> + @JsonProperty
> + String location; //newLocation
> + @JsonProperty
> + String oldLocation;
> +
> + public SentryJSONAlterTableMessage() {
> + }
> +
> + public SentryJSONAlterTableMessage(String server, String
> servicePrincipal, String db, String table,
> + Long timestamp, String
> oldLocation, String location) {
> + super(server, servicePrincipal, db, table, timestamp);
> + this.location = location;
> + this.oldLocation = oldLocation;
> + }
> +
> + public String getLocation() {
> + return location;
> + }
> + public String getOldLocation() {
> + return oldLocation;
> + }
> +
> + @Override
> + public String toString() {
> + return SentryJSONMessageDeserializer.serialize(this);
> + }
> +}
>
>
> http://git-wip-us.apache.org/repos/asf/sentry/blob/113c5eae/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateDatabaseMessage.java
> ----------------------------------------------------------------------
> diff --git
> a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateDatabaseMessage.java
> b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateDatabaseMessage.java
> new file mode 100644
> index 0000000..ba19cbe
> --- /dev/null
> +++
> b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateDatabaseMessage.java
> @@ -0,0 +1,44 @@
> +/**
> + * Licensed to the Apache Software Foundation (ASF) under one
> + * or more contributor license agreements. See the NOTICE file
> + * distributed with this work for additional information
> + * regarding copyright ownership. The ASF licenses this file
> + * to you under the Apache License, Version 2.0 (the
> + * "License"); you may not use this file except in compliance
> + * with the License. You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> + */
> +
> +package org.apache.sentry.binding.metastore.messaging.json;
> +
> +import org.apache.hive.hcatalog.messaging.json.JSONCreateDatabaseMessage;
> +import org.codehaus.jackson.annotate.JsonProperty;
> +
> +public class SentryJSONCreateDatabaseMessage extends
> JSONCreateDatabaseMessage {
> + @JsonProperty
> + String location;
> +
> + public SentryJSONCreateDatabaseMessage() {
> + }
> +
> + public SentryJSONCreateDatabaseMessage(String server, String
> servicePrincipal, String db, Long timestamp, String location) {
> + super(server, servicePrincipal, db, timestamp);
> + this.location = location;
> + }
> +
> + public String getLocation() {
> + return location;
> + }
> +
> + @Override
> + public String toString() {
> + return SentryJSONMessageDeserializer.serialize(this);
> + }
> +}
>
>
> http://git-wip-us.apache.org/repos/asf/sentry/blob/113c5eae/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateTableMessage.java
> ----------------------------------------------------------------------
> diff --git
> a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateTableMessage.java
> b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateTableMessage.java
> new file mode 100644
> index 0000000..57d11d2
> --- /dev/null
> +++
> b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateTableMessage.java
> @@ -0,0 +1,45 @@
> +/**
> + * Licensed to the Apache Software Foundation (ASF) under one
> + * or more contributor license agreements. See the NOTICE file
> + * distributed with this work for additional information
> + * regarding copyright ownership. The ASF licenses this file
> + * to you under the Apache License, Version 2.0 (the
> + * "License"); you may not use this file except in compliance
> + * with the License. You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> + */
> +
> +package org.apache.sentry.binding.metastore.messaging.json;
> +
> +import org.apache.hive.hcatalog.messaging.json.JSONCreateTableMessage;
> +import org.codehaus.jackson.annotate.JsonProperty;
> +
> +public class SentryJSONCreateTableMessage extends JSONCreateTableMessage {
> + @JsonProperty
> + String location;
> +
> + public SentryJSONCreateTableMessage() {
> + }
> +
> + public SentryJSONCreateTableMessage(String server, String
> servicePrincipal, String db, String table, Long timestamp, String location)
> {
> + super(server, servicePrincipal, db, table, timestamp);
> + this.location = location;
> + }
> +
> + public String getLocation() {
> + return location;
> + }
> +
> + @Override
> + public String toString() {
> + return SentryJSONMessageDeserializer.serialize(this);
> + }
> +
> +}
>
>
> http://git-wip-us.apache.org/repos/asf/sentry/blob/113c5eae/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropDatabaseMessage.java
> ----------------------------------------------------------------------
> diff --git
> a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropDatabaseMessage.java
> b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropDatabaseMessage.java
> new file mode 100644
> index 0000000..05f83f7
> --- /dev/null
> +++
> b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropDatabaseMessage.java
> @@ -0,0 +1,44 @@
> +/**
> + * Licensed to the Apache Software Foundation (ASF) under one
> + * or more contributor license agreements. See the NOTICE file
> + * distributed with this work for additional information
> + * regarding copyright ownership. The ASF licenses this file
> + * to you under the Apache License, Version 2.0 (the
> + * "License"); you may not use this file except in compliance
> + * with the License. You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> + */
> +
> +package org.apache.sentry.binding.metastore.messaging.json;
> +
> +import org.apache.hive.hcatalog.messaging.json.JSONDropDatabaseMessage;
> +import org.codehaus.jackson.annotate.JsonProperty;
> +
> +public class SentryJSONDropDatabaseMessage extends
> JSONDropDatabaseMessage{
> + @JsonProperty
> + String location;
> +
> + public SentryJSONDropDatabaseMessage() {
> + }
> +
> + public SentryJSONDropDatabaseMessage(String server, String
> servicePrincipal, String db, Long timestamp, String location) {
> + super(server, servicePrincipal, db, timestamp);
> + this.location = location;
> + }
> +
> + public String getLocation() {
> + return location;
> + }
> +
> + @Override
> + public String toString() {
> + return SentryJSONMessageDeserializer.serialize(this);
> + }
> +}
>
>
> http://git-wip-us.apache.org/repos/asf/sentry/blob/113c5eae/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropPartitionMessage.java
> ----------------------------------------------------------------------
> diff --git
> a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropPartitionMessage.java
> b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropPartitionMessage.java
> new file mode 100644
> index 0000000..2ab61f7
> --- /dev/null
> +++
> b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropPartitionMessage.java
> @@ -0,0 +1,49 @@
> +/**
> + * Licensed to the Apache Software Foundation (ASF) under one
> + * or more contributor license agreements. See the NOTICE file
> + * distributed with this work for additional information
> + * regarding copyright ownership. The ASF licenses this file
> + * to you under the Apache License, Version 2.0 (the
> + * "License"); you may not use this file except in compliance
> + * with the License. You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> + */
> +
> +package org.apache.sentry.binding.metastore.messaging.json;
> +
> +import org.apache.hive.hcatalog.messaging.json.JSONDropPartitionMessage;
> +import org.codehaus.jackson.annotate.JsonProperty;
> +
> +import java.util.List;
> +import java.util.Map;
> +
> +public class SentryJSONDropPartitionMessage extends
> JSONDropPartitionMessage {
> + @JsonProperty
> + String location;
> +
> + public SentryJSONDropPartitionMessage() {
> + }
> +
> + public SentryJSONDropPartitionMessage(String server, String
> servicePrincipal, String db, String table,
> + List<Map<String, String>>
> partitions, Long timestamp, String location) {
> + super(server, servicePrincipal, db, table, partitions, timestamp);
> + this.location = location;
> + }
> +
> + public String getLocation() {
> + return location;
> + }
> +
> + @Override
> + public String toString() {
> + return SentryJSONMessageDeserializer.serialize(this);
> + }
> +
> +}
>
>
> http://git-wip-us.apache.org/repos/asf/sentry/blob/113c5eae/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropTableMessage.java
> ----------------------------------------------------------------------
> diff --git
> a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropTableMessage.java
> b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropTableMessage.java
> new file mode 100644
> index 0000000..7005776
> --- /dev/null
> +++
> b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropTableMessage.java
> @@ -0,0 +1,45 @@
> +/**
> + * Licensed to the Apache Software Foundation (ASF) under one
> + * or more contributor license agreements. See the NOTICE file
> + * distributed with this work for additional information
> + * regarding copyright ownership. The ASF licenses this file
> + * to you under the Apache License, Version 2.0 (the
> + * "License"); you may not use this file except in compliance
> + * with the License. You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> + */
> +
> +package org.apache.sentry.binding.metastore.messaging.json;
> +
> +import org.apache.hive.hcatalog.messaging.json.JSONDropTableMessage;
> +import org.codehaus.jackson.annotate.JsonProperty;
> +
> +
> +public class SentryJSONDropTableMessage extends JSONDropTableMessage {
> + @JsonProperty
> + String location;
> +
> + public SentryJSONDropTableMessage() {
> + }
> +
> + public SentryJSONDropTableMessage(String server, String
> servicePrincipal, String db, String table, Long timestamp, String location)
> {
> + super(server, servicePrincipal, db, table, timestamp);
> + this.location = location;
> + }
> +
> + public String getLocation() {
> + return location;
> + }
> +
> + @Override
> + public String toString() {
> + return SentryJSONMessageDeserializer.serialize(this);
> + }
> +}
>
>
> http://git-wip-us.apache.org/repos/asf/sentry/blob/113c5eae/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageDeserializer.java
> ----------------------------------------------------------------------
> diff --git
> a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageDeserializer.java
> b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageDeserializer.java
> new file mode 100644
> index 0000000..b645c45
> --- /dev/null
> +++
> b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageDeserializer.java
> @@ -0,0 +1,110 @@
> +/**
> + * Licensed to the Apache Software Foundation (ASF) under one
> + * or more contributor license agreements. See the NOTICE file
> + * distributed with this work for additional information
> + * regarding copyright ownership. The ASF licenses this file
> + * to you under the Apache License, Version 2.0 (the
> + * "License"); you may not use this file except in compliance
> + * with the License. You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> + */
> +
> +package org.apache.sentry.binding.metastore.messaging.json;
> +
> +import org.apache.hive.hcatalog.messaging.*;
> +import org.codehaus.jackson.map.DeserializationConfig;
> +import org.codehaus.jackson.map.ObjectMapper;
> +
> +public class SentryJSONMessageDeserializer extends MessageDeserializer {
> + static ObjectMapper mapper = new ObjectMapper();
> +
> + public SentryJSONMessageDeserializer() {
> + }
> +
> + /**
> + * Method to de-serialize CreateDatabaseMessage instance.
> + */
> + public SentryJSONCreateDatabaseMessage
> getCreateDatabaseMessage(String messageBody) {
> + try {
> + return
> (SentryJSONCreateDatabaseMessage)mapper.readValue(messageBody,
> SentryJSONCreateDatabaseMessage.class);
> + } catch (Exception var3) {
> + throw new IllegalArgumentException("Could not construct
> SentryJSONCreateDatabaseMessage.", var3);
> + }
> + }
> +
> + public SentryJSONDropDatabaseMessage getDropDatabaseMessage(String
> messageBody) {
> + try {
> + return
> (SentryJSONDropDatabaseMessage)mapper.readValue(messageBody,
> SentryJSONDropDatabaseMessage.class);
> + } catch (Exception var3) {
> + throw new IllegalArgumentException("Could not construct
> SentryJSONDropDatabaseMessage.", var3);
> + }
> + }
> +
> + public SentryJSONCreateTableMessage getCreateTableMessage(String
> messageBody) {
> + try {
> + return
> (SentryJSONCreateTableMessage)mapper.readValue(messageBody,
> SentryJSONCreateTableMessage.class);
> + } catch (Exception var3) {
> + throw new IllegalArgumentException("Could not construct
> SentryJSONCreateTableMessage.", var3);
> + }
> + }
> +
> + public SentryJSONAlterTableMessage getAlterTableMessage(String
> messageBody) {
> + try {
> + return
> (SentryJSONAlterTableMessage)mapper.readValue(messageBody,
> SentryJSONAlterTableMessage.class);
> + } catch (Exception var3) {
> + throw new IllegalArgumentException("Could not construct
> SentryJSONAlterTableMessage.", var3);
> + }
> + }
> +
> + public SentryJSONDropTableMessage getDropTableMessage(String
> messageBody) {
> + try {
> + return
> (SentryJSONDropTableMessage)mapper.readValue(messageBody,
> SentryJSONDropTableMessage.class);
> + } catch (Exception var3) {
> + throw new IllegalArgumentException("Could not construct
> SentryJSONDropTableMessage.", var3);
> + }
> + }
> +
> + public SentryJSONAddPartitionMessage getAddPartitionMessage(String
> messageBody) {
> + try {
> + return
> (SentryJSONAddPartitionMessage)mapper.readValue(messageBody,
> SentryJSONAddPartitionMessage.class);
> + } catch (Exception var3) {
> + throw new IllegalArgumentException("Could not construct
> SentryJSONAddPartitionMessage.", var3);
> + }
> + }
> +
> + public SentryJSONAlterPartitionMessage
> getAlterPartitionMessage(String messageBody) {
> + try {
> + return
> (SentryJSONAlterPartitionMessage)mapper.readValue(messageBody,
> SentryJSONAlterPartitionMessage.class);
> + } catch (Exception var3) {
> + throw new IllegalArgumentException("Could not construct
> SentryJSONAlterPartitionMessage.", var3);
> + }
> + }
> +
> + public SentryJSONDropPartitionMessage getDropPartitionMessage(String
> messageBody) {
> + try {
> + return
> (SentryJSONDropPartitionMessage)mapper.readValue(messageBody,
> SentryJSONDropPartitionMessage.class);
> + } catch (Exception var3) {
> + throw new IllegalArgumentException("Could not construct
> SentryJSONDropPartitionMessage.", var3);
> + }
> + }
> +
> + static {
> +
> mapper.configure(DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES,
> false);
> + }
> +
> + public static String serialize(Object object) {
> + try {
> + return mapper.writeValueAsString(object);
> + }
> + catch (Exception exception) {
> + throw new IllegalArgumentException("Could not serialize: ",
> exception);
> + }
> + }
> +}
>
>
> http://git-wip-us.apache.org/repos/asf/sentry/blob/113c5eae/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageFactory.java
> ----------------------------------------------------------------------
> diff --git
> a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageFactory.java
> b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageFactory.java
> new file mode 100644
> index 0000000..00e7db8
> --- /dev/null
> +++
> b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageFactory.java
> @@ -0,0 +1,177 @@
> +/**
> + * Licensed to the Apache Software Foundation (ASF) under one
> + * or more contributor license agreements. See the NOTICE file
> + * distributed with this work for additional information
> + * regarding copyright ownership. The ASF licenses this file
> + * to you under the Apache License, Version 2.0 (the
> + * "License"); you may not use this file except in compliance
> + * with the License. You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> + */
> +package org.apache.sentry.binding.metastore.messaging.json;
> +
> +import org.apache.commons.logging.Log;
> +import org.apache.commons.logging.LogFactory;
> +import org.apache.hadoop.hive.common.classification.InterfaceAudience;
> +import org.apache.hadoop.hive.common.classification.InterfaceStability;
> +import org.apache.hadoop.hive.metastore.api.Database;
> +import org.apache.hadoop.hive.metastore.api.FieldSchema;
> +import org.apache.hadoop.hive.metastore.api.Partition;
> +import org.apache.hadoop.hive.metastore.api.Table;
> +import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy;
> +import org.apache.hive.hcatalog.messaging.*;
> +
> +import java.util.*;
> +
> +public class SentryJSONMessageFactory extends MessageFactory {
> + private static final Log LOG =
> LogFactory.getLog(SentryJSONMessageFactory.class.getName());
> + private static SentryJSONMessageDeserializer deserializer = new
> SentryJSONMessageDeserializer();
> + public SentryJSONMessageFactory() {
> + LOG.info("Using SentryJSONMessageFactory for building
> Notification log messages ");
> +
> + }
> + public MessageDeserializer getDeserializer() {
> + return deserializer;
> + }
> +
> + public String getVersion() {
> + return "0.1";
> + }
> +
> + public String getMessageFormat() {
> + return "json";
> + }
> +
> + public SentryJSONCreateDatabaseMessage
> buildCreateDatabaseMessage(Database db) {
> + return new SentryJSONCreateDatabaseMessage(HCAT_SERVER_URL,
> HCAT_SERVICE_PRINCIPAL, db.getName(),
> + Long.valueOf(this.now()), db.getLocationUri());
> + }
> + public SentryJSONDropDatabaseMessage
> buildDropDatabaseMessage(Database db) {
> + return new SentryJSONDropDatabaseMessage(HCAT_SERVER_URL,
> HCAT_SERVICE_PRINCIPAL, db.getName(),
> + Long.valueOf(this.now()), db.getLocationUri());
> + }
> +
> + public SentryJSONCreateTableMessage buildCreateTableMessage(Table
> table) {
> + return new SentryJSONCreateTableMessage(HCAT_SERVER_URL,
> HCAT_SERVICE_PRINCIPAL, table.getDbName(),
> + table.getTableName(), Long.valueOf(this.now()),
> table.getSd().getLocation());
> + }
> +
> + public SentryJSONAlterTableMessage buildAlterTableMessage(Table
> before, Table after) {
> + return new SentryJSONAlterTableMessage(HCAT_SERVER_URL,
> HCAT_SERVICE_PRINCIPAL, before.getDbName(),
> + before.getTableName(), Long.valueOf(this.now()),
> before.getSd().getLocation(), after.getSd().getLocation());
> + }
> +
> + public SentryJSONDropTableMessage buildDropTableMessage(Table table) {
> + return new SentryJSONDropTableMessage(HCAT_SERVER_URL,
> HCAT_SERVICE_PRINCIPAL, table.getDbName(),
> + table.getTableName(), Long.valueOf(this.now()),
> table.getSd().getLocation());
> + }
> +
> + public SentryJSONAddPartitionMessage buildAddPartitionMessage(Table
> table, List<Partition> partitions) {
> + return new SentryJSONAddPartitionMessage(HCAT_SERVER_URL,
> HCAT_SERVICE_PRINCIPAL, table.getDbName(),
> + table.getTableName(), getPartitionKeyValues(table,
> partitions), Long.valueOf(this.now()),
> + getPartitionLocations(partitions));
> + }
> +
> + private List<String> getPartitionLocations(List<Partition>
> partitions) {
> + List<String> paths = new ArrayList<String>();
> + for(Partition partition:partitions) {
> + paths.add(partition.getSd().getLocation());
> + }
> + return paths;
> + }
> +
> + //TODO: Not sure what is this used for. Need to investigate
> + private List<String> getPartitionLocations(PartitionSpecProxy
> partitionSpec) {
> + Iterator<Partition> iterator =
> partitionSpec.getPartitionIterator();
> + List<String> locations = new ArrayList<String>();
> + while(iterator.hasNext()) {
> + locations.add(iterator.next().getSd().getLocation());
> + }
> + return locations;
> + }
> +
> + @InterfaceAudience.LimitedPrivate({"Hive"})
> + @InterfaceStability.Evolving
> + public SentryJSONAddPartitionMessage buildAddPartitionMessage(Table
> table, PartitionSpecProxy partitionSpec) {
> + return new SentryJSONAddPartitionMessage(HCAT_SERVER_URL,
> HCAT_SERVICE_PRINCIPAL, table.getDbName(),
> + table.getTableName(), getPartitionKeyValues(table,
> partitionSpec), Long.valueOf(this.now()),
> + getPartitionLocations(partitionSpec));
> + }
> +
> + public SentryJSONAlterPartitionMessage
> buildAlterPartitionMessage(Partition before, Partition after) {
> + /*
> + f (partitionEvent.getOldPartition() != null) {
> + oldLoc = partitionEvent.getOldPartition().getSd().getLocation();
> + }
> + if (partitionEvent.getNewPartition() != null) {
> + newLoc = partitionEvent.getNewPartition().getSd().getLocation();
> + }
> +
> + if ((oldLoc != null) && (newLoc != null) && (!oldLoc.equals(newLoc)))
> {
> + String authzObj =
> + partitionEvent.getOldPartition().getDbName() + "."
> + + partitionEvent.getOldPartition().getTableName();
> + for (SentryMetastoreListenerPlugin plugin : sentryPlugins) {
> + plugin.renameAuthzObject(authzObj, oldLoc,
> + authzObj, newLoc);
> + }
> + }
> + * */
> + return new SentryJSONAlterPartitionMessage(HCAT_SERVER_URL,
> HCAT_SERVICE_PRINCIPAL, before.getDbName(),
> + before.getTableName(), before.getValues(),
> Long.valueOf(this.now()), before.getSd().getLocation(),
> + after.getSd().getLocation());
> + }
> +
> + public SentryJSONDropPartitionMessage buildDropPartitionMessage(Table
> table, Partition partition) {
> + return new SentryJSONDropPartitionMessage(HCAT_SERVER_URL,
> HCAT_SERVICE_PRINCIPAL, partition.getDbName(),
> + partition.getTableName(),
> Arrays.asList(getPartitionKeyValues(table, partition)),
> + Long.valueOf(this.now()),
> partition.getSd().getLocation());
> + }
> +
> + private static Map<String, String> getPartitionKeyValues(Table table,
> Partition partition) {
> + LinkedHashMap partitionKeys = new LinkedHashMap();
> +
> + for(int i = 0; i < table.getPartitionKeysSize(); ++i) {
> +
> partitionKeys.put(((FieldSchema)table.getPartitionKeys().get(i)).getName(),
> partition.getValues().get(i));
> + }
> +
> + return partitionKeys;
> + }
> +
> + private static List<Map<String, String>> getPartitionKeyValues(Table
> table, List<Partition> partitions) {
> + ArrayList partitionList = new ArrayList(partitions.size());
> + Iterator i$ = partitions.iterator();
> +
> + while(i$.hasNext()) {
> + Partition partition = (Partition)i$.next();
> + partitionList.add(getPartitionKeyValues(table, partition));
> + }
> +
> + return partitionList;
> + }
> +
> + @InterfaceAudience.LimitedPrivate({"Hive"})
> + @InterfaceStability.Evolving
> + private static List<Map<String, String>> getPartitionKeyValues(Table
> table, PartitionSpecProxy partitionSpec) {
> + ArrayList partitionList = new ArrayList();
> + PartitionSpecProxy.PartitionIterator iterator =
> partitionSpec.getPartitionIterator();
> +
> + while(iterator.hasNext()) {
> + Partition partition = (Partition)iterator.next();
> + partitionList.add(getPartitionKeyValues(table, partition));
> + }
> +
> + return partitionList;
> + }
> + //This is private in parent class
> + private long now() {
> + return System.currentTimeMillis() / 1000L;
> + }
> +}
>
>
> http://git-wip-us.apache.org/repos/asf/sentry/blob/113c5eae/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbPrivilegeCleanupOnDrop.java
> ----------------------------------------------------------------------
> diff --git
> a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbPrivilegeCleanupOnDrop.java
> b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbPrivilegeCleanupOnDrop.java
> index 767bcbe..439b9de 100644
> ---
> a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbPrivilegeCleanupOnDrop.java
> +++
> b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbPrivilegeCleanupOnDrop.java
> @@ -32,14 +32,13 @@ import java.util.ArrayList;
> import java.util.List;
>
> import
> org.apache.sentry.tests.e2e.hive.AbstractTestWithStaticConfiguration;
> -import org.junit.After;
> -import org.junit.Before;
> -import org.junit.BeforeClass;
> -import org.junit.Test;
> +import org.junit.*;
>
> import com.google.common.collect.Lists;
> import com.google.common.io.Resources;
>
> +
> +@Ignore("Ignoring until SENTRY-1321 is complete")
> public class TestDbPrivilegeCleanupOnDrop extends
> AbstractTestWithStaticConfiguration {
>
>
>
> http://git-wip-us.apache.org/repos/asf/sentry/blob/113c5eae/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java
> ----------------------------------------------------------------------
> diff --git
> a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java
> b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java
> index 2c4948e..7dc3d0f 100644
> ---
> a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java
> +++
> b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java
> @@ -515,7 +515,8 @@ public abstract class
> AbstractTestWithStaticConfiguration {
> } else {
>
> properties.put(HiveConf.ConfVars.METASTORE_EVENT_LISTENERS.varname,
> SentryMetastorePostEventListener.class.getName());
> -
> + properties.put("hcatalog.message.factory.impl.json",
> +
> "org.apache.sentry.binding.metastore.messaging.json.SentryJSONMessageFactory");
> }
> }
> }
>
>
> http://git-wip-us.apache.org/repos/asf/sentry/blob/113c5eae/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/AbstractMetastoreTestWithStaticConfiguration.java
> ----------------------------------------------------------------------
> diff --git
> a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/AbstractMetastoreTestWithStaticConfiguration.java
> b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/AbstractMetastoreTestWithStaticConfiguration.java
> index b72e317..567b4c8 100644
> ---
> a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/AbstractMetastoreTestWithStaticConfiguration.java
> +++
> b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/AbstractMetastoreTestWithStaticConfiguration.java
> @@ -200,6 +200,11 @@ public abstract class
> AbstractMetastoreTestWithStaticConfiguration extends
> client.createDatabase(db);
> }
>
> + public void dropMetastoreDBIfExists(HiveMetaStoreClient client, String
> dbName)
> + throws Exception {
> + client.dropDatabase(dbName, true, true, true);
> + }
> +
> public void execHiveSQLwithOverlay(final String sqlStmt,
> final String userName, Map<String, String> overLay) throws
> Exception {
> final HiveConf hiveConf = new HiveConf();
>
>