You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sentry.apache.org by ha...@apache.org on 2017/01/10 00:11:21 UTC
[2/2] sentry git commit: SENTRY-1499: Add feature flag for using
NotificationLog (Hao Hao, Reviewed by: Sravya Tirukkovalur, Kalyan Kalvagadda,
Alexander Kolbasov and Vamsee Yarlagadda)
SENTRY-1499: Add feature flag for using NotificationLog (Hao Hao, Reviewed by: Sravya Tirukkovalur, Kalyan Kalvagadda, Alexander Kolbasov and Vamsee Yarlagadda)
Change-Id: I8e817357b9c833f0aed0e3732b159c866d952f97
Project: http://git-wip-us.apache.org/repos/asf/sentry/repo
Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/b34a736a
Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/b34a736a
Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/b34a736a
Branch: refs/heads/sentry-ha-redesign
Commit: b34a736a64a60790651b39fd77d0a9ce39cd3c2a
Parents: 5e6da0c
Author: hahao <ha...@cloudera.com>
Authored: Mon Jan 9 15:52:29 2017 -0800
Committer: hahao <ha...@cloudera.com>
Committed: Mon Jan 9 16:01:19 2017 -0800
----------------------------------------------------------------------
.../SentryMetastorePostEventListener.java | 523 ++++++++++---------
...tastorePostEventListenerNotificationLog.java | 395 ++++++++++++++
.../sentry/service/thrift/SentryService.java | 22 +-
.../sentry/service/thrift/ServiceConstants.java | 2 +
.../AbstractTestWithStaticConfiguration.java | 16 +-
.../TestSentryListenerInBuiltDeserializer.java | 1 +
.../TestSentryListenerSentryDeserializer.java | 1 +
7 files changed, 702 insertions(+), 258 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sentry/blob/b34a736a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentryMetastorePostEventListener.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentryMetastorePostEventListener.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentryMetastorePostEventListener.java
index 75190c1..fdb6df4 100644
--- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentryMetastorePostEventListener.java
+++ b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentryMetastorePostEventListener.java
@@ -17,16 +17,15 @@
*/
package org.apache.sentry.binding.metastore;
-import java.util.concurrent.TimeUnit;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.MetaStoreEventListener;
-import org.apache.hadoop.hive.metastore.RawStore;
-import org.apache.hadoop.hive.metastore.RawStoreProxy;
-import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.NotificationEvent;
+import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.events.AddPartitionEvent;
import org.apache.hadoop.hive.metastore.events.AlterPartitionEvent;
import org.apache.hadoop.hive.metastore.events.AlterTableEvent;
@@ -35,360 +34,386 @@ import org.apache.hadoop.hive.metastore.events.CreateTableEvent;
import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent;
import org.apache.hadoop.hive.metastore.events.DropPartitionEvent;
import org.apache.hadoop.hive.metastore.events.DropTableEvent;
-import org.apache.hive.hcatalog.common.HCatConstants;
-import org.apache.sentry.binding.metastore.messaging.json.SentryJSONMessageFactory;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.sentry.core.common.exception.SentryUserException;
+import org.apache.sentry.binding.hive.conf.HiveAuthzConf;
+import org.apache.sentry.binding.hive.conf.HiveAuthzConf.AuthzConfVars;
+import org.apache.sentry.core.common.Authorizable;
+import org.apache.sentry.core.model.db.Database;
+import org.apache.sentry.core.model.db.Server;
+import org.apache.sentry.core.model.db.Table;
import org.apache.sentry.provider.db.SentryMetastoreListenerPlugin;
+import org.apache.sentry.provider.db.service.thrift.SentryPolicyServiceClient;
+import org.apache.sentry.service.thrift.SentryServiceClientFactory;
+import org.apache.sentry.service.thrift.ServiceConstants.ConfUtilties;
+import org.apache.sentry.service.thrift.ServiceConstants.ServerConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.commons.lang3.builder.ToStringBuilder;
-/*
-A HMS listener class which should ideally go into the transaction which persists the Hive metadata.
-This class writes all DDL events to the NotificationLog through rawstore.addNotificationEvent(event)
-This class is very similar to DbNotificationListener, except:
-1. It uses a custom SentryJSONMessageFactory which adds additional information to the message part of the event
- to avoid another round trip from the clients
-2. It handles the cases where actual operation has failed, and hence skips writing to the notification log.
-3. Has additional validations to make sure event has the required information.
-
-This can be replaced with DbNotificationListener in future and sentry's message factory can be plugged in if:
-- HIVE-14011 is fixed: Make MessageFactory truly pluggable
-- 2 and 3 above are handled in DbNotificationListener
-*/
+/**
+ * SentryMetastorePostEventListener class is HMS plugin for listening to
+ * all DDL events and deliver those events to Sentry server. This class
+ * sends all DDL events to the Sentry server through thrift API.
+ *
+ * In case any actual event fails, skipping deliver the event to Sentry server.
+ */
public class SentryMetastorePostEventListener extends MetaStoreEventListener {
private static final Logger LOGGER = LoggerFactory.getLogger(SentryMetastoreListenerPlugin.class);
- private RawStore rs;
- private HiveConf hiveConf;
- SentryJSONMessageFactory messageFactory;
-
- private static SentryMetastorePostEventListener.CleanerThread cleaner = null;
-
- //Same as DbNotificationListener to make the transition back easy
- private synchronized void init(HiveConf conf) {
- try {
- this.rs = RawStoreProxy.getProxy(conf, conf, conf.getVar(HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL), 999999);
- } catch (MetaException var3) {
- LOGGER.error("Unable to connect to raw store, notifications will not be tracked", var3);
- this.rs = null;
- }
+ private final HiveAuthzConf authzConf;
+ private final Server server;
- if(cleaner == null && this.rs != null) {
- cleaner = new SentryMetastorePostEventListener.CleanerThread(conf, this.rs);
- cleaner.start();
- }
- }
+ private List<SentryMetastoreListenerPlugin> sentryPlugins = new ArrayList<SentryMetastoreListenerPlugin>();
public SentryMetastorePostEventListener(Configuration config) {
super(config);
- // The code in MetastoreUtils.getMetaStoreListeners() that calls this looks for a constructor
- // with a Configuration parameter, so we have to declare config as Configuration. But it
- // actually passes a HiveConf, which we need. So we'll do this ugly down cast.
+
if (!(config instanceof HiveConf)) {
- String error = "Could not initialize Plugin - Configuration is not an instanceof HiveConf";
- LOGGER.error(error);
- throw new RuntimeException(error);
+ String error = "Could not initialize Plugin - Configuration is not an instanceof HiveConf";
+ LOGGER.error(error);
+ throw new RuntimeException(error);
+ }
+
+ authzConf = HiveAuthzConf.getAuthzConf((HiveConf)config);
+ server = new Server(authzConf.get(AuthzConfVars.AUTHZ_SERVER_NAME.getVar()));
+ Iterable<String> pluginClasses = ConfUtilties.CLASS_SPLITTER
+ .split(config.get(ServerConfig.SENTRY_METASTORE_PLUGINS, ServerConfig.SENTRY_METASTORE_PLUGINS_DEFAULT).trim());
+
+ try {
+ for (String pluginClassStr : pluginClasses) {
+ Class<?> clazz = config.getClassByName(pluginClassStr);
+ if (!SentryMetastoreListenerPlugin.class.isAssignableFrom(clazz)) {
+ throw new IllegalArgumentException("Class \\"
+ + pluginClassStr + "\\ is not a "
+ + SentryMetastoreListenerPlugin.class.getName());
+ }
+ SentryMetastoreListenerPlugin plugin = (SentryMetastoreListenerPlugin) clazz
+ .getConstructor(Configuration.class, Configuration.class)
+ .newInstance(config, authzConf);
+ sentryPlugins.add(plugin);
+ }
+ } catch (Exception e) {
+ LOGGER.error("Could not initialize HMS Plugin: SentryMetastorePostEventListener !!", e);
+ throw new RuntimeException(e);
}
- hiveConf = (HiveConf)config;
- messageFactory = new SentryJSONMessageFactory();
- init(hiveConf);
}
@Override
- public void onCreateDatabase(CreateDatabaseEvent dbEvent)
- throws MetaException {
+ public void onCreateTable (CreateTableEvent tableEvent) throws MetaException {
- // do not write to Notification log if the operation has failed
- if (!dbEvent.getStatus()) {
- LOGGER.info("Skipping writing to NotificationLog as the Create database event failed");
+ // don't sync paths/privileges if the operation has failed
+ if (!tableEvent.getStatus()) {
+ LOGGER.debug("Skip sync paths/privileges with Sentry server for onCreateTable event," +
+ " since the operation failed. \n");
return;
}
- String location = dbEvent.getDatabase().getLocationUri();
- if (location == null || location.isEmpty()) {
- throw new SentryMalformedEventException("CreateDatabaseEvent has invalid location", dbEvent);
+ if (tableEvent.getTable().getSd().getLocation() != null) {
+ String authzObj = tableEvent.getTable().getDbName() + "."
+ + tableEvent.getTable().getTableName();
+ String path = tableEvent.getTable().getSd().getLocation();
+ for (SentryMetastoreListenerPlugin plugin : sentryPlugins) {
+ plugin.addPath(authzObj, path);
+ }
}
- String dbName = dbEvent.getDatabase().getName();
- if (dbName == null || dbName.isEmpty()) {
- throw new SentryMalformedEventException("CreateDatabaseEvent has invalid dbName", dbEvent);
+
+ // drop the privileges on the given table, in case if anything was left
+ // behind during the drop
+ if (!syncWithPolicyStore(AuthzConfVars.AUTHZ_SYNC_CREATE_WITH_POLICY_STORE)) {
+ return;
}
- NotificationEvent event = new NotificationEvent(0L, now(), HCatConstants.HCAT_CREATE_DATABASE_EVENT,
- messageFactory.buildCreateDatabaseMessage(dbEvent.getDatabase()).toString());
- event.setDbName(dbName);
- this.enqueue(event);
+ dropSentryTablePrivilege(tableEvent.getTable().getDbName(),
+ tableEvent.getTable().getTableName());
}
@Override
- public void onDropDatabase(DropDatabaseEvent dbEvent) throws MetaException {
+ public void onDropTable(DropTableEvent tableEvent) throws MetaException {
- // do not write to Notification log if the operation has failed
- if (!dbEvent.getStatus()) {
- LOGGER.info("Skipping writing to NotificationLog as the Drop database event failed");
+ // don't sync paths/privileges if the operation has failed
+ if (!tableEvent.getStatus()) {
+ LOGGER.debug("Skip syncing paths/privileges with Sentry server for onDropTable event," +
+ " since the operation failed. \n");
+ return;
+ }
+
+ if (tableEvent.getTable().getSd().getLocation() != null) {
+ String authzObj = tableEvent.getTable().getDbName() + "."
+ + tableEvent.getTable().getTableName();
+ for (SentryMetastoreListenerPlugin plugin : sentryPlugins) {
+ plugin.removeAllPaths(authzObj, null);
+ }
+ }
+ // drop the privileges on the given table
+ if (!syncWithPolicyStore(AuthzConfVars.AUTHZ_SYNC_DROP_WITH_POLICY_STORE)) {
return;
}
- String dbName = dbEvent.getDatabase().getName();
- if (dbName == null || dbName.isEmpty()) {
- throw new SentryMalformedEventException("DropDatabaseEvent has invalid dbName", dbEvent);
+ if (!tableEvent.getStatus()) {
+ return;
}
- NotificationEvent event = new NotificationEvent(0L, now(), HCatConstants.HCAT_DROP_DATABASE_EVENT,
- messageFactory.buildDropDatabaseMessage(dbEvent.getDatabase()).toString());
- event.setDbName(dbName);
- this.enqueue(event);
+ dropSentryTablePrivilege(tableEvent.getTable().getDbName(),
+ tableEvent.getTable().getTableName());
}
@Override
- public void onCreateTable (CreateTableEvent tableEvent) throws MetaException {
+ public void onCreateDatabase(CreateDatabaseEvent dbEvent)
+ throws MetaException {
- // do not write to Notification log if the operation has failed
- if (!tableEvent.getStatus()) {
- LOGGER.info("Skipping writing to NotificationLog as the Create table event failed");
+ // don't sync paths/privileges if the operation has failed
+ if (!dbEvent.getStatus()) {
+ LOGGER.debug("Skip syncing paths/privileges with Sentry server for onCreateDatabase event," +
+ " since the operation failed. \n");
return;
}
- String dbName = tableEvent.getTable().getDbName();
- if (dbName == null || dbName.isEmpty()) {
- throw new SentryMalformedEventException("CreateTableEvent has invalid dbName", tableEvent);
- }
- String tableName = tableEvent.getTable().getTableName();
- if (tableName == null || tableName.isEmpty()) {
- throw new SentryMalformedEventException("CreateTableEvent has invalid tableName", tableEvent);
- }
- // Create table event should also contain a location.
- // But, Create view also generates a Create table event, but it does not have a location.
- // Create view is identified by the tableType. But turns out tableType is not set in some cases.
- // We assume that tableType is set for all create views.
- //TODO: Location can be null/empty, handle that in HMSFollower
- String tableType = tableEvent.getTable().getTableType();
- if(!(tableType != null && tableType.equals(TableType.VIRTUAL_VIEW.name()))) {
- if (tableType == null) {
- LOGGER.warn("TableType is null, assuming it is not TableType.VIRTUAL_VIEW: tableEvent", tableEvent);
- }
- String location = tableEvent.getTable().getSd().getLocation();
- if (location == null || location.isEmpty()) {
- throw new SentryMalformedEventException("CreateTableEvent has invalid location", tableEvent);
+ if (dbEvent.getDatabase().getLocationUri() != null) {
+ String authzObj = dbEvent.getDatabase().getName();
+ String path = dbEvent.getDatabase().getLocationUri();
+ for (SentryMetastoreListenerPlugin plugin : sentryPlugins) {
+ plugin.addPath(authzObj, path);
}
}
- NotificationEvent event = new NotificationEvent(0L, now(), HCatConstants.HCAT_CREATE_TABLE_EVENT,
- messageFactory.buildCreateTableMessage(tableEvent.getTable()).toString());
- event.setDbName(dbName);
- event.setTableName(tableName);
- this.enqueue(event);
+ // drop the privileges on the database, in case anything left behind during
+ // last drop db
+ if (!syncWithPolicyStore(AuthzConfVars.AUTHZ_SYNC_CREATE_WITH_POLICY_STORE)) {
+ return;
+ }
+
+ dropSentryDbPrivileges(dbEvent.getDatabase().getName());
}
+ /**
+ * Drop the privileges on the database. Note that child tables will be
+ * dropped individually by client, so we just need to handle the removing
+ * the db privileges. The table drop should cleanup the table privileges.
+ */
@Override
- public void onDropTable(DropTableEvent tableEvent) throws MetaException {
+ public void onDropDatabase(DropDatabaseEvent dbEvent) throws MetaException {
- // do not write to Notification log if the operation has failed
- if (!tableEvent.getStatus()) {
- LOGGER.info("Skipping writing to NotificationLog as the Drop table event failed");
+ // don't sync paths/privileges if the operation has failed
+ if (!dbEvent.getStatus()) {
+ LOGGER.debug("Skip syncing paths/privileges with Sentry server for onDropDatabase event," + " since the operation failed. \n");
return;
}
- String dbName = tableEvent.getTable().getDbName();
- if (dbName == null || dbName.isEmpty()) {
- throw new SentryMalformedEventException("DropTableEvent has invalid dbName", tableEvent);
+ String authzObj = dbEvent.getDatabase().getName();
+ for (SentryMetastoreListenerPlugin plugin : sentryPlugins) {
+ List<String> tNames = dbEvent.getHandler().get_all_tables(authzObj);
+ plugin.removeAllPaths(authzObj, tNames);
}
- String tableName = tableEvent.getTable().getTableName();
- if (tableName == null || tableName.isEmpty()) {
- throw new SentryMalformedEventException("DropTableEvent has invalid tableName", tableEvent);
+ if (!syncWithPolicyStore(AuthzConfVars.AUTHZ_SYNC_DROP_WITH_POLICY_STORE)) {
+ return;
}
- NotificationEvent event = new NotificationEvent(0L, now(), HCatConstants.HCAT_DROP_TABLE_EVENT,
- messageFactory.buildDropTableMessage(tableEvent.getTable()).toString());
- event.setDbName(dbName);
- event.setTableName(tableName);
- this.enqueue(event);
+ dropSentryDbPrivileges(dbEvent.getDatabase().getName());
}
+ /**
+ * Adjust the privileges when table is renamed
+ */
@Override
public void onAlterTable (AlterTableEvent tableEvent) throws MetaException {
- // do not write to Notification log if the operation has failed
+ // don't sync privileges if the operation has failed
if (!tableEvent.getStatus()) {
- LOGGER.info("Skipping writing to NotificationLog as the Alter table event failed");
+ LOGGER.debug("Skip syncing privileges with Sentry server for onAlterTable event," +
+ " since the operation failed. \n");
return;
}
- String dbName = tableEvent.getNewTable().getDbName();
- if (dbName == null || dbName.isEmpty()) {
- throw new SentryMalformedEventException("AlterTableEvent's newTable has invalid dbName", tableEvent);
- }
- String tableName = tableEvent.getNewTable().getTableName();
- if (tableName == null || tableName.isEmpty()) {
- throw new SentryMalformedEventException("AlterTableEvent's newTable has invalid tableName", tableEvent);
- }
- dbName = tableEvent.getOldTable().getDbName();
- if (dbName == null || dbName.isEmpty()) {
- throw new SentryMalformedEventException("AlterTableEvent's oldTable has invalid dbName", tableEvent);
+ String oldLoc = null, newLoc = null;
+
+ org.apache.hadoop.hive.metastore.api.Table oldTal = tableEvent.getOldTable();
+ org.apache.hadoop.hive.metastore.api.Table newTal = tableEvent.getNewTable();
+
+ if(oldTal != null && oldTal.getSd() !=null) {
+ oldLoc = oldTal.getSd().getLocation();
}
- tableName = tableEvent.getOldTable().getTableName();
- if (tableName == null || tableName.isEmpty()) {
- throw new SentryMalformedEventException("AlterTableEvent's oldTable has invalid tableName", tableEvent);
+ if (newTal != null && newTal.getSd() != null) {
+ newLoc = newTal.getSd().getLocation();
}
- //Alter view also generates an alter table event, but it does not have a location
- //TODO: Handle this case in Sentry
- if(!tableEvent.getOldTable().getTableType().equals(TableType.VIRTUAL_VIEW.name())) {
- String location = tableEvent.getNewTable().getSd().getLocation();
- if (location == null || location.isEmpty()) {
- throw new SentryMalformedEventException("AlterTableEvent's newTable has invalid location", tableEvent);
- }
- location = tableEvent.getOldTable().getSd().getLocation();
- if (location == null || location.isEmpty()) {
- throw new SentryMalformedEventException("AlterTableEvent's oldTable has invalid location", tableEvent);
- }
+ if(oldLoc != null && newLoc != null && !oldLoc.equals(newLoc)) {
+ String oldDbName = tableEvent.getOldTable().getDbName();
+ String oldTbName = tableEvent.getOldTable().getTableName();
+ String newTbName = tableEvent.getNewTable().getTableName();
+ String newDbName = tableEvent.getNewTable().getDbName();
+ renameSentryTablePrivilege(oldDbName, oldTbName, oldLoc, newDbName, newTbName, newLoc);
}
-
- NotificationEvent event = new NotificationEvent(0L, now(), HCatConstants.HCAT_ALTER_TABLE_EVENT,
- messageFactory.buildAlterTableMessage(tableEvent.getOldTable(), tableEvent.getNewTable()).toString());
- event.setDbName(tableEvent.getNewTable().getDbName());
- event.setTableName(tableEvent.getNewTable().getTableName());
- this.enqueue(event);
}
@Override
public void onAlterPartition(AlterPartitionEvent partitionEvent)
- throws MetaException {
+ throws MetaException {
- // do not write to Notification log if the operation has failed
+ // don't sync privileges if the operation has failed
if (!partitionEvent.getStatus()) {
- LOGGER.info("Skipping writing to NotificationLog as the Alter partition event failed");
+ LOGGER.debug("Skip syncing privileges with Sentry server for onAlterPartition event," +
+ " since the operation failed. \n");
return;
}
- String dbName = partitionEvent.getNewPartition().getDbName();
- if (dbName == null || dbName.isEmpty()) {
- throw new SentryMalformedEventException("AlterPartitionEvent's newPartition has invalid dbName", partitionEvent);
+ String oldLoc = null, newLoc = null;
+ if (partitionEvent.getOldPartition() != null) {
+ oldLoc = partitionEvent.getOldPartition().getSd().getLocation();
}
- String tableName = partitionEvent.getNewPartition().getTableName();
- if (tableName == null || tableName.isEmpty()) {
- throw new SentryMalformedEventException("AlterPartitionEvent's newPartition has invalid tableName", partitionEvent);
+ if (partitionEvent.getNewPartition() != null) {
+ newLoc = partitionEvent.getNewPartition().getSd().getLocation();
}
- //TODO: Need more validations, but it is tricky as there are many variations and validations change for each one
- // Alter partition Location
- // Alter partition property
- // Any more?
-
- NotificationEvent event = new NotificationEvent(0L, now(), HCatConstants.HCAT_ALTER_PARTITION_EVENT,
- messageFactory.buildAlterPartitionMessage(partitionEvent.getOldPartition(), partitionEvent.getNewPartition()).toString());
-
- event.setDbName(partitionEvent.getNewPartition().getDbName());
- event.setTableName(partitionEvent.getNewPartition().getTableName());
- this.enqueue(event);
+ if (oldLoc != null && newLoc != null && !oldLoc.equals(newLoc)) {
+ String authzObj =
+ partitionEvent.getOldPartition().getDbName() + "."
+ + partitionEvent.getOldPartition().getTableName();
+ for (SentryMetastoreListenerPlugin plugin : sentryPlugins) {
+ plugin.renameAuthzObject(authzObj, oldLoc,
+ authzObj, newLoc);
+ }
+ }
}
@Override
public void onAddPartition(AddPartitionEvent partitionEvent)
- throws MetaException {
+ throws MetaException {
- // do not write to Notification log if the operation has failed
+ // don't sync path if the operation has failed
if (!partitionEvent.getStatus()) {
- LOGGER.info("Skipping writing to NotificationLog as the Add partition event failed");
+ LOGGER.debug("Skip syncing path with Sentry server for onAddPartition event," + " since the operation failed. \n");
return;
}
- String dbName = partitionEvent.getTable().getDbName();
- if (dbName == null || dbName.isEmpty()) {
- throw new SentryMalformedEventException("AddPartitionEvent has invalid dbName", partitionEvent);
- }
- String tableName = partitionEvent.getTable().getTableName();
- if (tableName == null || tableName.isEmpty()) {
- throw new SentryMalformedEventException("AddPartitionEvent's newPartition has invalid tableName", partitionEvent);
+ for (Partition part : partitionEvent.getPartitions()) {
+ if (part.getSd() != null && part.getSd().getLocation() != null) {
+ String authzObj = part.getDbName() + "." + part.getTableName();
+ String path = part.getSd().getLocation();
+ for (SentryMetastoreListenerPlugin plugin : sentryPlugins) {
+ plugin.addPath(authzObj, path);
+ }
+ }
}
-
- //TODO: Need more validations?
-
- NotificationEvent event = new NotificationEvent(0L, now(), HCatConstants.HCAT_ADD_PARTITION_EVENT,
- messageFactory.buildAddPartitionMessage(partitionEvent.getTable(), partitionEvent.getPartitions()).toString());
-
- event.setDbName(partitionEvent.getTable().getDbName());
- event.setTableName(partitionEvent.getTable().getTableName());
- this.enqueue(event);
+ super.onAddPartition(partitionEvent);
}
@Override
public void onDropPartition(DropPartitionEvent partitionEvent)
- throws MetaException {
+ throws MetaException {
- // do not write to Notification log if the operation has failed
+ // don't sync path if the operation has failed
if (!partitionEvent.getStatus()) {
- LOGGER.info("Skipping writing to NotificationLog as the Drop partition event failed");
+ LOGGER.debug("Skip syncing path with Sentry server for onDropPartition event," +
+ " since the operation failed. \n");
return;
}
- NotificationEvent event = new NotificationEvent(0L, now(), HCatConstants.HCAT_DROP_PARTITION_EVENT,
- messageFactory.buildDropPartitionMessage(partitionEvent.getTable(), partitionEvent.getPartition()).toString());
- //TODO: Why is this asymmetric with add partitions(s)?
- // Seems like adding multiple partitions generate a single event
- // where as single partition drop generated an event?
-
- event.setDbName(partitionEvent.getTable().getDbName());
- event.setTableName(partitionEvent.getTable().getTableName());
- this.enqueue(event);
+ String authzObj = partitionEvent.getTable().getDbName() + "."
+ + partitionEvent.getTable().getTableName();
+ String path = partitionEvent.getPartition().getSd().getLocation();
+ for (SentryMetastoreListenerPlugin plugin : sentryPlugins) {
+ plugin.removePath(authzObj, path);
+ }
+ super.onDropPartition(partitionEvent);
}
- private int now() {
- long millis = System.currentTimeMillis();
- millis /= 1000;
- if (millis > Integer.MAX_VALUE) {
- LOGGER.warn("We've passed max int value in seconds since the epoch, " +
- "all notification times will be the same!");
- return Integer.MAX_VALUE;
+ private SentryPolicyServiceClient getSentryServiceClient()
+ throws MetaException {
+ try {
+ return SentryServiceClientFactory.create(authzConf);
+ } catch (Exception e) {
+ throw new MetaException("Failed to connect to Sentry service "
+ + e.getMessage());
}
- return (int)millis;
}
- //Same as DbNotificationListener to make the transition back easy
- private void enqueue(NotificationEvent event) {
- if(this.rs != null) {
- this.rs.addNotificationEvent(event);
- } else {
- LOGGER.warn("Dropping event " + event + " since notification is not running.");
+ private void dropSentryDbPrivileges(String dbName) throws MetaException {
+ List<Authorizable> authorizableTable = new ArrayList<Authorizable>();
+ authorizableTable.add(server);
+ authorizableTable.add(new Database(dbName));
+ try {
+ dropSentryPrivileges(authorizableTable);
+ } catch (SentryUserException e) {
+ throw new MetaException("Failed to remove Sentry policies for drop DB "
+ + dbName + " Error: " + e.getMessage());
+ } catch (IOException e) {
+ throw new MetaException("Failed to find local user " + e.getMessage());
}
+
}
- //Same as DbNotificationListener to make the transition back easy
- private static class CleanerThread extends Thread {
- private RawStore rs;
- private int ttl;
+ private void dropSentryTablePrivilege(String dbName, String tabName)
+ throws MetaException {
+ List<Authorizable> authorizableTable = new ArrayList<Authorizable>();
+ authorizableTable.add(server);
+ authorizableTable.add(new Database(dbName));
+ authorizableTable.add(new Table(tabName));
- CleanerThread(HiveConf conf, RawStore rs) {
- super("CleanerThread");
- this.rs = rs;
- this.setTimeToLive(conf.getTimeVar(HiveConf.ConfVars.METASTORE_EVENT_DB_LISTENER_TTL, TimeUnit.SECONDS));
- this.setDaemon(true);
+ try {
+ dropSentryPrivileges(authorizableTable);
+ } catch (SentryUserException e) {
+ throw new MetaException(
+ "Failed to remove Sentry policies for drop table " + dbName + "."
+ + tabName + " Error: " + e.getMessage());
+ } catch (IOException e) {
+ throw new MetaException("Failed to find local user " + e.getMessage());
}
- public void run() {
- while(true) {
- this.rs.cleanNotificationEvents(this.ttl);
+ }
+ private void dropSentryPrivileges(
+ List<? extends Authorizable> authorizableTable)
+ throws SentryUserException, IOException, MetaException {
+ String requestorUserName = UserGroupInformation.getCurrentUser()
+ .getShortUserName();
+ SentryPolicyServiceClient sentryClient = getSentryServiceClient();
+ sentryClient.dropPrivileges(requestorUserName, authorizableTable);
+
+ // Close the connection after dropping privileges is done.
+ sentryClient.close();
+ }
- try {
- Thread.sleep(60000L);
- } catch (InterruptedException var2) {
- LOGGER.info("Cleaner thread sleep interupted", var2);
- }
+ private void renameSentryTablePrivilege(String oldDbName, String oldTabName,
+ String oldPath, String newDbName, String newTabName, String newPath)
+ throws MetaException {
+ List<Authorizable> oldAuthorizableTable = new ArrayList<Authorizable>();
+ oldAuthorizableTable.add(server);
+ oldAuthorizableTable.add(new Database(oldDbName));
+ oldAuthorizableTable.add(new Table(oldTabName));
+
+ List<Authorizable> newAuthorizableTable = new ArrayList<Authorizable>();
+ newAuthorizableTable.add(server);
+ newAuthorizableTable.add(new Database(newDbName));
+ newAuthorizableTable.add(new Table(newTabName));
+
+ if (!oldTabName.equalsIgnoreCase(newTabName)
+ && syncWithPolicyStore(AuthzConfVars.AUTHZ_SYNC_ALTER_WITH_POLICY_STORE)) {
+
+ SentryPolicyServiceClient sentryClient = getSentryServiceClient();
+
+ try {
+ String requestorUserName = UserGroupInformation.getCurrentUser()
+ .getShortUserName();
+ sentryClient.renamePrivileges(requestorUserName, oldAuthorizableTable, newAuthorizableTable);
+ } catch (SentryUserException e) {
+ throw new MetaException(
+ "Failed to remove Sentry policies for rename table " + oldDbName
+ + "." + oldTabName + "to " + newDbName + "." + newTabName
+ + " Error: " + e.getMessage());
+ } catch (IOException e) {
+ throw new MetaException("Failed to find local user " + e.getMessage());
+ } finally {
+
+ // Close the connection after renaming privileges is done.
+ sentryClient.close();
}
}
-
- public void setTimeToLive(long configTtl) {
- if(configTtl > 2147483647L) {
- this.ttl = 2147483647;
- } else {
- this.ttl = (int)configTtl;
- }
-
+ // The HDFS plugin needs to know if it's a path change (set location)
+ for (SentryMetastoreListenerPlugin plugin : sentryPlugins) {
+ plugin.renameAuthzObject(oldDbName + "." + oldTabName, oldPath,
+ newDbName + "." + newTabName, newPath);
}
}
- private class SentryMalformedEventException extends MetaException {
- SentryMalformedEventException(String msg, Object event) {
- //toString is not implemented in Event classes,
- // hence using reflection to print the details of the Event object.
- super(msg + "Event: " + ToStringBuilder.reflectionToString(event));
- }
+
+ private boolean syncWithPolicyStore(AuthzConfVars syncConfVar) {
+ return Boolean.parseBoolean(authzConf.get(syncConfVar.getVar(), "true"));
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/sentry/blob/b34a736a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentryMetastorePostEventListenerNotificationLog.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentryMetastorePostEventListenerNotificationLog.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentryMetastorePostEventListenerNotificationLog.java
new file mode 100644
index 0000000..58470d6
--- /dev/null
+++ b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentryMetastorePostEventListenerNotificationLog.java
@@ -0,0 +1,395 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sentry.binding.metastore;
+
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.base.Strings;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.MetaStoreEventListener;
+import org.apache.hadoop.hive.metastore.RawStore;
+import org.apache.hadoop.hive.metastore.RawStoreProxy;
+import org.apache.hadoop.hive.metastore.TableType;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.NotificationEvent;
+import org.apache.hadoop.hive.metastore.events.AddPartitionEvent;
+import org.apache.hadoop.hive.metastore.events.AlterPartitionEvent;
+import org.apache.hadoop.hive.metastore.events.AlterTableEvent;
+import org.apache.hadoop.hive.metastore.events.CreateDatabaseEvent;
+import org.apache.hadoop.hive.metastore.events.CreateTableEvent;
+import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent;
+import org.apache.hadoop.hive.metastore.events.DropPartitionEvent;
+import org.apache.hadoop.hive.metastore.events.DropTableEvent;
+import org.apache.hive.hcatalog.common.HCatConstants;
+import org.apache.sentry.binding.metastore.messaging.json.SentryJSONMessageFactory;
+import org.apache.sentry.provider.db.SentryMetastoreListenerPlugin;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.lang3.builder.ToStringBuilder;
+/*
+A HMS listener class which should ideally go into the transaction which persists the Hive metadata.
+This class writes all DDL events to the NotificationLog through rawstore.addNotificationEvent(event)
+This class is very similar to DbNotificationListener, except:
+1. It uses a custom SentryJSONMessageFactory which adds additional information to the message part of the event
+ to avoid another round trip from the clients
+2. It handles the cases where actual operation has failed, and hence skips writing to the notification log.
+3. Has additional validations to make sure event has the required information.
+
+This can be replaced with DbNotificationListener in future and sentry's message factory can be plugged in if:
+- HIVE-14011 is fixed: Make MessageFactory truly pluggable
+- 2 and 3 above are handled in DbNotificationListener
+*/
+
+public class SentryMetastorePostEventListenerNotificationLog extends MetaStoreEventListener {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(SentryMetastoreListenerPlugin.class);
+ private RawStore rs;
+ private HiveConf hiveConf;
+ SentryJSONMessageFactory messageFactory;
+
+ private static SentryMetastorePostEventListenerNotificationLog.CleanerThread cleaner = null;
+
+ //Same as DbNotificationListener to make the transition back easy
+ private synchronized void init(HiveConf conf) {
+ try {
+ this.rs = RawStoreProxy.getProxy(conf, conf, conf.getVar(HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL), 999999);
+ } catch (MetaException var3) {
+ LOGGER.error("Unable to connect to raw store, notifications will not be tracked", var3);
+ this.rs = null;
+ }
+
+ if(cleaner == null && this.rs != null) {
+ cleaner = new SentryMetastorePostEventListenerNotificationLog.CleanerThread(conf, this.rs);
+ cleaner.start();
+ }
+ }
+
+ public SentryMetastorePostEventListenerNotificationLog(Configuration config) {
+ super(config);
+ // The code in MetastoreUtils.getMetaStoreListeners() that calls this looks for a constructor
+ // with a Configuration parameter, so we have to declare config as Configuration. But it
+ // actually passes a HiveConf, which we need. So we'll do this ugly down cast.
+ if (!(config instanceof HiveConf)) {
+ String error = "Could not initialize Plugin - Configuration is not an instanceof HiveConf";
+ LOGGER.error(error);
+ throw new RuntimeException(error);
+ }
+ hiveConf = (HiveConf)config;
+ messageFactory = new SentryJSONMessageFactory();
+ init(hiveConf);
+ }
+
+ @Override
+ public void onCreateDatabase(CreateDatabaseEvent dbEvent)
+ throws MetaException {
+
+ // do not write to Notification log if the operation has failed
+ if (!dbEvent.getStatus()) {
+ LOGGER.info("Skipping writing to NotificationLog as the Create database event failed");
+ return;
+ }
+
+ String location = dbEvent.getDatabase().getLocationUri();
+ if (Strings.isNullOrEmpty(location)) {
+ throw new SentryMalformedEventException("CreateDatabaseEvent has invalid location", dbEvent);
+ }
+ String dbName = dbEvent.getDatabase().getName();
+ if (Strings.isNullOrEmpty(dbName)) {
+ throw new SentryMalformedEventException("CreateDatabaseEvent has invalid dbName", dbEvent);
+ }
+
+ NotificationEvent event = new NotificationEvent(0L, now(), HCatConstants.HCAT_CREATE_DATABASE_EVENT,
+ messageFactory.buildCreateDatabaseMessage(dbEvent.getDatabase()).toString());
+ event.setDbName(dbName);
+ this.enqueue(event);
+ }
+
+ @Override
+ public void onDropDatabase(DropDatabaseEvent dbEvent) throws MetaException {
+
+ // do not write to Notification log if the operation has failed
+ if (!dbEvent.getStatus()) {
+ LOGGER.info("Skipping writing to NotificationLog as the Drop database event failed");
+ return;
+ }
+
+ String dbName = dbEvent.getDatabase().getName();
+ if (dbName == null || dbName.isEmpty()) {
+ throw new SentryMalformedEventException("DropDatabaseEvent has invalid dbName", dbEvent);
+ }
+
+ NotificationEvent event = new NotificationEvent(0L, now(), HCatConstants.HCAT_DROP_DATABASE_EVENT,
+ messageFactory.buildDropDatabaseMessage(dbEvent.getDatabase()).toString());
+ event.setDbName(dbName);
+ this.enqueue(event);
+ }
+
+ @Override
+ public void onCreateTable (CreateTableEvent tableEvent) throws MetaException {
+
+ // do not write to Notification log if the operation has failed
+ if (!tableEvent.getStatus()) {
+ LOGGER.info("Skipping writing to NotificationLog as the Create table event failed");
+ return;
+ }
+
+ String dbName = tableEvent.getTable().getDbName();
+ if (dbName == null || dbName.isEmpty()) {
+ throw new SentryMalformedEventException("CreateTableEvent has invalid dbName", tableEvent);
+ }
+ String tableName = tableEvent.getTable().getTableName();
+ if (tableName == null || tableName.isEmpty()) {
+ throw new SentryMalformedEventException("CreateTableEvent has invalid tableName", tableEvent);
+ }
+ // Create table event should also contain a location.
+ // But, Create view also generates a Create table event, but it does not have a location.
+ // Create view is identified by the tableType. But turns out tableType is not set in some cases.
+ // We assume that tableType is set for all create views.
+ //TODO: Location can be null/empty, handle that in HMSFollower
+ String tableType = tableEvent.getTable().getTableType();
+ if(!(tableType != null && tableType.equals(TableType.VIRTUAL_VIEW.name()))) {
+ if (tableType == null) {
+ LOGGER.warn("TableType is null, assuming it is not TableType.VIRTUAL_VIEW: tableEvent", tableEvent);
+ }
+ String location = tableEvent.getTable().getSd().getLocation();
+ if (location == null || location.isEmpty()) {
+ throw new SentryMalformedEventException("CreateTableEvent has invalid location", tableEvent);
+ }
+ }
+ NotificationEvent event = new NotificationEvent(0L, now(), HCatConstants.HCAT_CREATE_TABLE_EVENT,
+ messageFactory.buildCreateTableMessage(tableEvent.getTable()).toString());
+ event.setDbName(dbName);
+ event.setTableName(tableName);
+ this.enqueue(event);
+ }
+
+ @Override
+ public void onDropTable(DropTableEvent tableEvent) throws MetaException {
+
+ // do not write to Notification log if the operation has failed
+ if (!tableEvent.getStatus()) {
+ LOGGER.info("Skipping writing to NotificationLog as the Drop table event failed");
+ return;
+ }
+
+ String dbName = tableEvent.getTable().getDbName();
+ if (dbName == null || dbName.isEmpty()) {
+ throw new SentryMalformedEventException("DropTableEvent has invalid dbName", tableEvent);
+ }
+ String tableName = tableEvent.getTable().getTableName();
+ if (tableName == null || tableName.isEmpty()) {
+ throw new SentryMalformedEventException("DropTableEvent has invalid tableName", tableEvent);
+ }
+
+ NotificationEvent event = new NotificationEvent(0L, now(), HCatConstants.HCAT_DROP_TABLE_EVENT,
+ messageFactory.buildDropTableMessage(tableEvent.getTable()).toString());
+ event.setDbName(dbName);
+ event.setTableName(tableName);
+ this.enqueue(event);
+ }
+
+ @Override
+ public void onAlterTable (AlterTableEvent tableEvent) throws MetaException {
+
+ // do not write to Notification log if the operation has failed
+ if (!tableEvent.getStatus()) {
+ LOGGER.info("Skipping writing to NotificationLog as the Alter table event failed");
+ return;
+ }
+
+ String dbName = tableEvent.getNewTable().getDbName();
+ if (dbName == null || dbName.isEmpty()) {
+ throw new SentryMalformedEventException("AlterTableEvent's newTable has invalid dbName", tableEvent);
+ }
+ String tableName = tableEvent.getNewTable().getTableName();
+ if (tableName == null || tableName.isEmpty()) {
+ throw new SentryMalformedEventException("AlterTableEvent's newTable has invalid tableName", tableEvent);
+ }
+ dbName = tableEvent.getOldTable().getDbName();
+ if (dbName == null || dbName.isEmpty()) {
+ throw new SentryMalformedEventException("AlterTableEvent's oldTable has invalid dbName", tableEvent);
+ }
+ tableName = tableEvent.getOldTable().getTableName();
+ if (tableName == null || tableName.isEmpty()) {
+ throw new SentryMalformedEventException("AlterTableEvent's oldTable has invalid tableName", tableEvent);
+ }
+ //Alter view also generates an alter table event, but it does not have a location
+ //TODO: Handle this case in Sentry
+ if(!tableEvent.getOldTable().getTableType().equals(TableType.VIRTUAL_VIEW.name())) {
+ String location = tableEvent.getNewTable().getSd().getLocation();
+ if (location == null || location.isEmpty()) {
+ throw new SentryMalformedEventException("AlterTableEvent's newTable has invalid location", tableEvent);
+ }
+ location = tableEvent.getOldTable().getSd().getLocation();
+ if (location == null || location.isEmpty()) {
+ throw new SentryMalformedEventException("AlterTableEvent's oldTable has invalid location", tableEvent);
+ }
+ }
+
+ NotificationEvent event = new NotificationEvent(0L, now(), HCatConstants.HCAT_ALTER_TABLE_EVENT,
+ messageFactory.buildAlterTableMessage(tableEvent.getOldTable(), tableEvent.getNewTable()).toString());
+ event.setDbName(tableEvent.getNewTable().getDbName());
+ event.setTableName(tableEvent.getNewTable().getTableName());
+ this.enqueue(event);
+ }
+
+ @Override
+ public void onAlterPartition(AlterPartitionEvent partitionEvent)
+ throws MetaException {
+
+ // do not write to Notification log if the operation has failed
+ if (!partitionEvent.getStatus()) {
+ LOGGER.info("Skipping writing to NotificationLog as the Alter partition event failed");
+ return;
+ }
+
+ String dbName = partitionEvent.getNewPartition().getDbName();
+ if (dbName == null || dbName.isEmpty()) {
+ throw new SentryMalformedEventException("AlterPartitionEvent's newPartition has invalid dbName", partitionEvent);
+ }
+ String tableName = partitionEvent.getNewPartition().getTableName();
+ if (tableName == null || tableName.isEmpty()) {
+ throw new SentryMalformedEventException("AlterPartitionEvent's newPartition has invalid tableName", partitionEvent);
+ }
+
+ //TODO: Need more validations, but it is tricky as there are many variations and validations change for each one
+ // Alter partition Location
+ // Alter partition property
+ // Any more?
+
+ NotificationEvent event = new NotificationEvent(0L, now(), HCatConstants.HCAT_ALTER_PARTITION_EVENT,
+ messageFactory.buildAlterPartitionMessage(partitionEvent.getOldPartition(), partitionEvent.getNewPartition()).toString());
+
+ event.setDbName(partitionEvent.getNewPartition().getDbName());
+ event.setTableName(partitionEvent.getNewPartition().getTableName());
+ this.enqueue(event);
+ }
+
+ @Override
+ public void onAddPartition(AddPartitionEvent partitionEvent)
+ throws MetaException {
+
+ // do not write to Notification log if the operation has failed
+ if (!partitionEvent.getStatus()) {
+ LOGGER.info("Skipping writing to NotificationLog as the Add partition event failed");
+ return;
+ }
+
+ String dbName = partitionEvent.getTable().getDbName();
+ if (dbName == null || dbName.isEmpty()) {
+ throw new SentryMalformedEventException("AddPartitionEvent has invalid dbName", partitionEvent);
+ }
+ String tableName = partitionEvent.getTable().getTableName();
+ if (tableName == null || tableName.isEmpty()) {
+ throw new SentryMalformedEventException("AddPartitionEvent's newPartition has invalid tableName", partitionEvent);
+ }
+
+ //TODO: Need more validations?
+
+ NotificationEvent event = new NotificationEvent(0L, now(), HCatConstants.HCAT_ADD_PARTITION_EVENT,
+ messageFactory.buildAddPartitionMessage(partitionEvent.getTable(), partitionEvent.getPartitions()).toString());
+
+ event.setDbName(partitionEvent.getTable().getDbName());
+ event.setTableName(partitionEvent.getTable().getTableName());
+ this.enqueue(event);
+ }
+
+ @Override
+ public void onDropPartition(DropPartitionEvent partitionEvent)
+ throws MetaException {
+
+ // do not write to Notification log if the operation has failed
+ if (!partitionEvent.getStatus()) {
+ LOGGER.info("Skipping writing to NotificationLog as the Drop partition event failed");
+ return;
+ }
+
+ NotificationEvent event = new NotificationEvent(0L, now(), HCatConstants.HCAT_DROP_PARTITION_EVENT,
+ messageFactory.buildDropPartitionMessage(partitionEvent.getTable(), partitionEvent.getPartition()).toString());
+ //TODO: Why is this asymmetric with add partitions(s)?
+ // Seems like adding multiple partitions generate a single event
+ // where as single partition drop generated an event?
+
+ event.setDbName(partitionEvent.getTable().getDbName());
+ event.setTableName(partitionEvent.getTable().getTableName());
+ this.enqueue(event);
+ }
+
+ private int now() {
+ long millis = System.currentTimeMillis();
+ millis /= 1000;
+ if (millis > Integer.MAX_VALUE) {
+ LOGGER.warn("We've passed max int value in seconds since the epoch, " +
+ "all notification times will be the same!");
+ return Integer.MAX_VALUE;
+ }
+ return (int)millis;
+ }
+
+ //Same as DbNotificationListener to make the transition back easy
+ private void enqueue(NotificationEvent event) {
+ if(this.rs != null) {
+ this.rs.addNotificationEvent(event);
+ } else {
+ LOGGER.warn("Dropping event " + event + " since notification is not running.");
+ }
+ }
+
+ //Same as DbNotificationListener to make the transition back easy
+ private static class CleanerThread extends Thread {
+ private RawStore rs;
+ private int ttl;
+
+ CleanerThread(HiveConf conf, RawStore rs) {
+ super("CleanerThread");
+ this.rs = rs;
+ this.setTimeToLive(conf.getTimeVar(HiveConf.ConfVars.METASTORE_EVENT_DB_LISTENER_TTL, TimeUnit.SECONDS));
+ this.setDaemon(true);
+ }
+
+ public void run() {
+ while(true) {
+ this.rs.cleanNotificationEvents(this.ttl);
+
+ try {
+ Thread.sleep(60000L);
+ } catch (InterruptedException var2) {
+ LOGGER.info("Cleaner thread sleep interupted", var2);
+ }
+ }
+ }
+
+ public void setTimeToLive(long configTtl) {
+ if(configTtl > 2147483647L) {
+ this.ttl = 2147483647;
+ } else {
+ this.ttl = (int)configTtl;
+ }
+
+ }
+ }
+ private class SentryMalformedEventException extends MetaException {
+ SentryMalformedEventException(String msg, Object event) {
+ //toString is not implemented in Event classes,
+ // hence using reflection to print the details of the Event object.
+ super(msg + "Event: " + ToStringBuilder.reflectionToString(event));
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/sentry/blob/b34a736a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java
index bd4ada4..a89d3f6 100644
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java
@@ -95,6 +95,7 @@ public class SentryService implements Callable, SigUtils.SigListener {
private SentryWebServer sentryWebServer;
private final long maxMessageSize;
private final LeaderStatusMonitor leaderMonitor;
+ private final boolean notificationLogEnabled;
public SentryService(Configuration conf) throws Exception {
this.conf = conf;
@@ -150,14 +151,21 @@ public class SentryService implements Callable, SigUtils.SigListener {
});
this.leaderMonitor = LeaderStatusMonitor.getLeaderStatusMonitor(conf);
webServerPort = conf.getInt(ServerConfig.SENTRY_WEB_PORT, ServerConfig.SENTRY_WEB_PORT_DEFAULT);
- try {
- hmsFollowerExecutor = Executors.newScheduledThreadPool(1);
- hmsFollowerExecutor.scheduleAtFixedRate(new HMSFollower(conf),
- 60000, 500, TimeUnit.MILLISECONDS);
- } catch(Exception e) {
- //TODO: Handle
- LOGGER.error("Could not start HMSFollower");
+
+ notificationLogEnabled = conf.getBoolean(ServerConfig.SENTRY_NOTIFICATION_LOG_ENABLED,
+ ServerConfig.SENTRY_NOTIFICATION_LOG_ENABLED_DEFAULT);
+
+ if (notificationLogEnabled) {
+ try {
+ hmsFollowerExecutor = Executors.newScheduledThreadPool(1);
+ hmsFollowerExecutor.scheduleAtFixedRate(new HMSFollower(conf),
+ 60000, 500, TimeUnit.MILLISECONDS);
+ } catch (Exception e) {
+ //TODO: Handle
+ LOGGER.error("Could not start HMSFollower");
+ }
}
+
status = Status.NOT_STARTED;
// Enable signal handler for HA leader/follower status if configured
http://git-wip-us.apache.org/repos/asf/sentry/blob/b34a736a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/ServiceConstants.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/ServiceConstants.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/ServiceConstants.java
index 139d038..866ebc6 100644
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/ServiceConstants.java
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/ServiceConstants.java
@@ -165,6 +165,8 @@ public class ServiceConstants {
.put("javax.jdo.option.Multithreaded", "true")
.build();
+ public static final String SENTRY_NOTIFICATION_LOG_ENABLED = "sentry.otification.log.enabled";
+ public static final boolean SENTRY_NOTIFICATION_LOG_ENABLED_DEFAULT = false;
public static final String SENTRY_WEB_ENABLE = "sentry.service.web.enable";
public static final Boolean SENTRY_WEB_ENABLE_DEFAULT = false;
public static final String SENTRY_WEB_PORT = "sentry.service.web.port";
http://git-wip-us.apache.org/repos/asf/sentry/blob/b34a736a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java
index 61b24fa..47be188 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java
@@ -54,6 +54,7 @@ import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.sentry.binding.hive.SentryHiveAuthorizationTaskFactoryImpl;
+import org.apache.sentry.binding.metastore.SentryMetastorePostEventListenerNotificationLog;
import org.apache.sentry.binding.metastore.SentryMetastorePostEventListener;
import org.apache.sentry.core.model.db.DBModelAction;
import org.apache.sentry.core.model.db.DBModelAuthorizable;
@@ -156,6 +157,7 @@ public abstract class AbstractTestWithStaticConfiguration {
private static final String EXTERNAL_SENTRY_SERVICE = "sentry.e2etest.external.sentry";
protected static final String EXTERNAL_HIVE_LIB = "sentry.e2etest.hive.lib";
private static final String ENABLE_SENTRY_HA = "sentry.e2etest.enable.service.ha";
+ private static final String ENABLE_NOTIFICATION_LOG = "sentry.e2etest.enable.notification.log";
protected static boolean policyOnHdfs = false;
protected static boolean useSentryService = false;
@@ -179,6 +181,7 @@ public abstract class AbstractTestWithStaticConfiguration {
protected static SentrySrv sentryServer;
protected static Configuration sentryConf;
protected static boolean enableSentryHA = false;
+ protected static boolean enableNotificationLog = false;
protected static Context context;
protected final String semanticException = "SemanticException No valid privileges";
@@ -312,6 +315,10 @@ public abstract class AbstractTestWithStaticConfiguration {
enableSentryHA = true;
}
+ if ("true".equalsIgnoreCase(System.getProperty(ENABLE_NOTIFICATION_LOG, "false"))) {
+ enableNotificationLog = true;
+ }
+
if (enableHiveConcurrency) {
properties.put(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "true");
properties.put(HiveConf.ConfVars.HIVE_TXN_MANAGER.varname,
@@ -514,8 +521,13 @@ public abstract class AbstractTestWithStaticConfiguration {
properties.put(HiveConf.ConfVars.METASTORE_EVENT_LISTENERS.varname,
"org.apache.hive.hcatalog.listener.DbNotificationListener");
} else {
- properties.put(HiveConf.ConfVars.METASTORE_EVENT_LISTENERS.varname,
- SentryMetastorePostEventListener.class.getName());
+ if (enableNotificationLog) {
+ properties.put(HiveConf.ConfVars.METASTORE_EVENT_LISTENERS.varname,
+ SentryMetastorePostEventListenerNotificationLog.class.getName());
+ } else {
+ properties.put(HiveConf.ConfVars.METASTORE_EVENT_LISTENERS.varname,
+ SentryMetastorePostEventListener.class.getName());
+ }
properties.put("hcatalog.message.factory.impl.json",
"org.apache.sentry.binding.metastore.messaging.json.SentryJSONMessageFactory");
}
http://git-wip-us.apache.org/repos/asf/sentry/blob/b34a736a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestSentryListenerInBuiltDeserializer.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestSentryListenerInBuiltDeserializer.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestSentryListenerInBuiltDeserializer.java
index c4be62d..6ab3391 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestSentryListenerInBuiltDeserializer.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestSentryListenerInBuiltDeserializer.java
@@ -31,6 +31,7 @@ public class TestSentryListenerInBuiltDeserializer extends TestDBNotificationLis
public static void setupTestStaticConfiguration() throws Exception {
setMetastoreListener = true;
useDbNotificationListener = false;
+ enableNotificationLog = true;
beforeClass();
}
}
http://git-wip-us.apache.org/repos/asf/sentry/blob/b34a736a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestSentryListenerSentryDeserializer.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestSentryListenerSentryDeserializer.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestSentryListenerSentryDeserializer.java
index 6f1886f..86a8964 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestSentryListenerSentryDeserializer.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestSentryListenerSentryDeserializer.java
@@ -53,6 +53,7 @@ public class TestSentryListenerSentryDeserializer extends AbstractMetastoreTestW
public static void setupTestStaticConfiguration() throws Exception {
setMetastoreListener = true;
useDbNotificationListener = false;
+ enableNotificationLog = true;
AbstractMetastoreTestWithStaticConfiguration.setupTestStaticConfiguration();
setupClass();
}