You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sentry.apache.org by sr...@apache.org on 2016/08/16 00:26:08 UTC
[1/2] sentry git commit: Sentry-1321: Implement HMSFollower in Sentry
service which reads the NotificationLog entries (Sravya Tirukkovalur,
Reviewed by: Hao Hao) (Append)
Repository: sentry
Updated Branches:
refs/heads/sentry-ha-redesign 2d5ed9984 -> 908072d66
http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/PoolClientInvocationHandler.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/PoolClientInvocationHandler.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/PoolClientInvocationHandler.java
index a35bf1d..353d461 100644
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/PoolClientInvocationHandler.java
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/PoolClientInvocationHandler.java
@@ -20,11 +20,13 @@ package org.apache.sentry.service.thrift;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.pool2.PooledObjectFactory;
import org.apache.commons.pool2.impl.AbandonedConfig;
import org.apache.commons.pool2.impl.GenericObjectPool;
import org.apache.commons.pool2.impl.GenericObjectPoolConfig;
import org.apache.hadoop.conf.Configuration;
+import org.apache.sentry.core.common.exception.SentryStandbyException;
import org.apache.sentry.core.common.exception.SentryUserException;
import org.apache.sentry.provider.db.service.thrift.SentryPolicyServiceClient;
import org.apache.sentry.service.thrift.ServiceConstants.ClientConfig;
@@ -33,63 +35,273 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * The PoolClientInvocationHandler is a proxy class for handling thrift call. For every thrift call,
- * get the instance of SentryPolicyServiceBaseClient from the commons-pool, and return the instance
- * to the commons-pool after complete the call. For any exception with the call, discard the
- * instance and create a new one added to the commons-pool. Then, get the instance and do the call
- * again. For the thread safe, the commons-pool will manage the connection pool, and every thread
- * can get the connection by borrowObject() and return the connection to the pool by returnObject().
+ * The PoolClientInvocationHandler is a proxy class for handling thrift
+ * call. For every thrift call, get the instance of
+ * SentryPolicyServiceBaseClient from the commons-pool, and return the instance
+ * to the commons-pool after complete the call. For any exception with the call,
+ * discard the instance and create a new one added to the commons-pool. Then,
+ * get the instance and do the call again. For the thread safe, the commons-pool
+ * will manage the connection pool, and every thread can get the connection by
+ * borrowObject() and return the connection to the pool by returnObject().
*/
public class PoolClientInvocationHandler extends SentryClientInvocationHandler {
+ private static final Logger LOGGER =
+ LoggerFactory.getLogger(PoolClientInvocationHandler.class);
- private static final Logger LOGGER = LoggerFactory.getLogger(PoolClientInvocationHandler.class);
+ private static final String POOL_EXCEPTION_MESSAGE = "Pool exception occurred ";
private final Configuration conf;
- private PooledObjectFactory<SentryPolicyServiceClient> poolFactory;
- private GenericObjectPool<SentryPolicyServiceClient> pool;
- private GenericObjectPoolConfig poolConfig;
- private int connectionRetryTotal;
- private static final String POOL_EXCEPTION_MESSAGE = "Pool exception occured ";
+ /**
+ * The configuration to use for our object pools.
+ * Null if we are not using object pools.
+ */
+ private final GenericObjectPoolConfig poolConfig;
+
+ /**
+ * The total number of connection retries to attempt per endpoint.
+ */
+ private final int connectionRetryTotal;
+
+ /**
+ * The configured sentry servers.
+ */
+ private final Endpoint[] endpoints;
+
+ /**
+ * The endpoint which we are currently using. This can be read without any locks.
+ * It must be written while holding the endpoints lock.
+ */
+ private volatile int freshestEndpointIdx = 0;
+
+ private class Endpoint {
+ /**
+ * The server address or hostname.
+ */
+ private final String addr;
+
+ /**
+ * The server port.
+ */
+ private final int port;
+
+ /**
+ * The server's poolFactory used to create new clients.
+ */
+ private final PooledObjectFactory<SentryPolicyServiceClient> poolFactory;
+
+ /**
+ * The server's pool of cached clients.
+ */
+ private final GenericObjectPool<SentryPolicyServiceClient> pool;
+
+ Endpoint(String addr, int port) {
+ this.addr = addr;
+ this.port = port;
+ this.poolFactory = new SentryServiceClientPoolFactory(addr, port, conf);
+ this.pool = new GenericObjectPool<SentryPolicyServiceClient>(
+ this.poolFactory, poolConfig, new AbandonedConfig());
+ }
+
+ GenericObjectPool<SentryPolicyServiceClient> getPool() {
+ return pool;
+ }
+
+ String getEndPointStr() {
+ return new String("endpoint at [address " + addr + ", port " + port + "]");
+ }
+ }
public PoolClientInvocationHandler(Configuration conf) throws Exception {
this.conf = conf;
- readConfiguration();
- poolFactory = new SentryServiceClientPoolFactory(conf);
- pool = new GenericObjectPool<SentryPolicyServiceClient>(poolFactory, poolConfig, new AbandonedConfig());
+
+ this.poolConfig = new GenericObjectPoolConfig();
+ // config the pool size for commons-pool
+ this.poolConfig.setMaxTotal(conf.getInt(ClientConfig.SENTRY_POOL_MAX_TOTAL,
+ ClientConfig.SENTRY_POOL_MAX_TOTAL_DEFAULT));
+ this.poolConfig.setMinIdle(conf.getInt(ClientConfig.SENTRY_POOL_MIN_IDLE,
+ ClientConfig.SENTRY_POOL_MIN_IDLE_DEFAULT));
+ this.poolConfig.setMaxIdle(conf.getInt(ClientConfig.SENTRY_POOL_MAX_IDLE,
+ ClientConfig.SENTRY_POOL_MAX_IDLE_DEFAULT));
+
+ // get the retry number for reconnecting service
+ this.connectionRetryTotal = conf.getInt(ClientConfig.SENTRY_POOL_RETRY_TOTAL,
+ ClientConfig.SENTRY_POOL_RETRY_TOTAL_DEFAULT);
+
+ String hostsAndPortsStr = conf.get(ClientConfig.SERVER_RPC_ADDRESS);
+ if (hostsAndPortsStr == null) {
+ throw new RuntimeException("Config key " +
+ ClientConfig.SERVER_RPC_ADDRESS + " is required");
+ }
+ int defaultPort = conf.getInt(ClientConfig.SERVER_RPC_PORT,
+ ClientConfig.SERVER_RPC_PORT_DEFAULT);
+ String[] hostsAndPorts = hostsAndPortsStr.split(",");
+ String[] hosts = new String[hostsAndPorts.length];
+ int[] ports = new int[hostsAndPorts.length];
+ parseHostPortStrings(hostsAndPortsStr, hostsAndPorts, hosts,
+ ports, defaultPort);
+ this.endpoints = new Endpoint[hostsAndPorts.length];
+ for (int i = 0; i < this.endpoints.length; i++) {
+ this.endpoints[i] = new Endpoint(hosts[i], ports[i]);
+ LOGGER.info("Initiate sentry sever endpoint: hostname " + hosts[i] + ", port " + ports[i]);
+ }
+ }
+
+ @VisibleForTesting
+ /**
+ * Utility function for parsing host and port strings. Expected form should be
+ * (host:port). The hostname could be in ipv6 style. Port number can be empty
+ * and will be default to defaultPort.
+ */
+ static protected void parseHostPortStrings(String hostsAndPortsStr,
+ String[] hostsAndPorts, String[] hosts, int[] ports,
+ int defaultPort) {
+ int i = -1;
+ for (String hostAndPort: hostsAndPorts) {
+ i++;
+ hostAndPort = hostAndPort.trim();
+ if (hostAndPort.isEmpty()) {
+ throw new RuntimeException("Cannot handle empty server RPC address " +
+ "in component " + (i + 1) + " of " + hostsAndPortsStr);
+ }
+ int colonIdx = hostAndPort.lastIndexOf(":");
+ if (colonIdx == -1) {
+ // There is no colon in the host+port string.
+ // That means the port is left unspecified, and should be set to
+ // the default.
+ hosts[i] = hostAndPort;
+ ports[i] = defaultPort;
+ continue;
+ }
+ int rightBracketIdx = hostAndPort.indexOf(']', colonIdx);
+ if (rightBracketIdx != -1) {
+ // If there is a right bracket that occurs after the colon, the
+ // colon we found is part of an ipv6 address like this:
+ // [::1]. That means we only have the host part, not the port part.
+ hosts[i] = hostAndPort.substring(0, rightBracketIdx);
+ ports[i] = defaultPort;
+ continue;
+ }
+ // We have a host:port string, where the part after colon should be
+ // the port.
+ hosts[i] = hostAndPort.substring(0, colonIdx);
+ String portStr = hostAndPort.substring(colonIdx+1);
+ try {
+ ports[i] = Integer.valueOf(portStr);
+ } catch (NumberFormatException e) {
+ throw new RuntimeException("Cannot parse port string " + portStr +
+ "in component " + (i + 1) + " of " + hostsAndPortsStr);
+ }
+ if ((ports[i] < 0) || (ports[i] > 65535)) {
+ throw new RuntimeException("Invalid port number given for " + portStr +
+ "in component " + (i + 1) + " of " + hostsAndPortsStr);
+ }
+ }
+ // Strip the brackets off of hostnames and ip addresses enclosed in square
+ // brackets. This is to support ipv6-style [address]:port addressing.
+ for (int j = 0; j < hosts.length; j++) {
+ if ((hosts[j].startsWith("[")) && (hosts[j].endsWith("]"))) {
+ hosts[j] = hosts[j].substring(1, hosts[j].length() - 1);
+ }
+ }
}
@Override
- public Object invokeImpl(Object proxy, Method method, Object[] args) throws Exception {
+ public Object invokeImpl(Object proxy, Method method, Object[] args)
+ throws Exception {
int retryCount = 0;
- Object result = null;
- while (retryCount < connectionRetryTotal) {
+ /**
+ * The maximum number of retries that we will do. Each endpoint gets its
+ * own set of retries.
+ */
+ int retryLimit = connectionRetryTotal * endpoints.length;
+
+ /**
+ * The index of the endpoint to use.
+ */
+ int endpointIdx = freshestEndpointIdx;
+
+ /**
+ * A list of exceptions from each endpoint. This starts as null to avoid
+ * memory allocation in the common case where there is no error.
+ */
+ Exception exc[] = null;
+
+ Object ret = null;
+
+ while (retryCount < retryLimit) {
+ GenericObjectPool<SentryPolicyServiceClient> pool =
+ endpoints[endpointIdx].getPool();
try {
- // The wapper here is for the retry of thrift call, the default retry number is 3.
- result = invokeFromPool(method, args);
+ if ((exc != null) &&
+ (exc[endpointIdx] instanceof TTransportException)) {
+ // If there was a TTransportException last time we tried to contact
+ // this endpoint, attempt to create a new connection before we try
+ // again.
+ synchronized (endpoints) {
+ // If there has room, create new instance and add it to the
+ // commons-pool. This instance will be returned first from the
+ // commons-pool, because the configuration is LIFO.
+ if (pool.getNumIdle() + pool.getNumActive() < pool.getMaxTotal()) {
+ pool.addObject();
+ }
+ }
+ }
+ // Try to make the RPC.
+ ret = invokeFromPool(method, args, pool);
break;
- } catch (TTransportException e) {
- // TTransportException means there has connection problem, create a new connection and try
- // again. Get the lock of pool and add new connection.
- synchronized (pool) {
- // If there has room, create new instance and add it to the commons-pool, this instance
- // will be back first from the commons-pool because the configuration is LIFO.
- if (pool.getNumIdle() + pool.getNumActive() < pool.getMaxTotal()) {
- pool.addObject();
+ } catch (SentryStandbyException | TTransportException e) {
+ if (exc == null) {
+ exc = new Exception[endpoints.length];
+ }
+ exc[endpointIdx] = e;
+ }
+
+ Exception lastExc = exc[endpointIdx];
+ synchronized (endpoints) {
+ int curFreshestEndpointIdx = freshestEndpointIdx;
+ if (curFreshestEndpointIdx == endpointIdx) {
+ curFreshestEndpointIdx =
+ (curFreshestEndpointIdx + 1) % endpoints.length;
+ freshestEndpointIdx = curFreshestEndpointIdx;
+ }
+ endpointIdx = curFreshestEndpointIdx;
+ }
+ // Increase the retry num, and throw the exception if can't retry again.
+ retryCount++;
+ if (retryCount == connectionRetryTotal) {
+ boolean allStandby = true, allUnreachable = true;
+ for (int i = 0; i < exc.length; i++) {
+ if (exc[i] instanceof SentryStandbyException) {
+ allUnreachable = false;
+ LOGGER.error("Sentry server " + endpoints[endpointIdx].getEndPointStr()
+ + " is in standby mode");
+ } else {
+ allStandby = false;
+ LOGGER.error("Sentry server " + endpoints[endpointIdx].getEndPointStr()
+ + " is in unreachable.");
}
}
- // Increase the retry num, and throw the exception if can't retry again.
- retryCount++;
- if (retryCount == connectionRetryTotal) {
- throw new SentryUserException(e.getMessage(), e);
+ if (allStandby) {
+ throw new SentryStandbyException("All sentry servers are in " +
+ "standby mode.", lastExc);
+ } else if (allUnreachable) {
+ throw new SentryUserException("All sentry servers are unreachable. " +
+ "Diagnostics is needed for unreachable servers.",
+ lastExc);
+ } else {
+ throw new SentryUserException("All reachable servers are standby. " +
+ "Diagnostics is needed for unreachable servers.",
+ lastExc);
}
}
}
- return result;
+ return ret;
}
- private Object invokeFromPool(Method method, Object[] args) throws Exception {
+ private Object invokeFromPool(Method method, Object[] args,
+ GenericObjectPool<SentryPolicyServiceClient> pool) throws Exception {
Object result = null;
SentryPolicyServiceClient client;
try {
@@ -106,7 +318,9 @@ public class PoolClientInvocationHandler extends SentryClientInvocationHandler {
// Get the target exception, check if SentryUserException or TTransportException is wrapped.
// TTransportException means there has connection problem with the pool.
Throwable targetException = e.getCause();
- if (targetException instanceof SentryUserException) {
+ if (targetException instanceof SentryStandbyException) {
+ throw (SentryStandbyException)targetException;
+ } else if (targetException instanceof SentryUserException) {
Throwable sentryTargetException = targetException.getCause();
// If there has connection problem, eg, invalid connection if the service restarted,
// sentryTargetException instanceof TTransportException = true.
@@ -134,21 +348,12 @@ public class PoolClientInvocationHandler extends SentryClientInvocationHandler {
@Override
public void close() {
- try {
- pool.close();
- } catch (Exception e) {
- LOGGER.debug(POOL_EXCEPTION_MESSAGE, e);
+ for (int i = 0; i < endpoints.length; i++) {
+ try {
+ endpoints[i].getPool().close();
+ } catch (Exception e) {
+ LOGGER.debug(POOL_EXCEPTION_MESSAGE, e);
+ }
}
}
-
- private void readConfiguration() {
- poolConfig = new GenericObjectPoolConfig();
- // config the pool size for commons-pool
- poolConfig.setMaxTotal(conf.getInt(ClientConfig.SENTRY_POOL_MAX_TOTAL, ClientConfig.SENTRY_POOL_MAX_TOTAL_DEFAULT));
- poolConfig.setMinIdle(conf.getInt(ClientConfig.SENTRY_POOL_MIN_IDLE, ClientConfig.SENTRY_POOL_MIN_IDLE_DEFAULT));
- poolConfig.setMaxIdle(conf.getInt(ClientConfig.SENTRY_POOL_MAX_IDLE, ClientConfig.SENTRY_POOL_MAX_IDLE_DEFAULT));
- // get the retry number for reconnecting service
- connectionRetryTotal = conf.getInt(ClientConfig.SENTRY_POOL_RETRY_TOTAL,
- ClientConfig.SENTRY_POOL_RETRY_TOTAL_DEFAULT);
- }
}
http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceClientFactory.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceClientFactory.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceClientFactory.java
index 56d774b..9e90af8 100644
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceClientFactory.java
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceClientFactory.java
@@ -24,7 +24,6 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.sentry.provider.db.service.thrift.SentryPolicyServiceClient;
import org.apache.sentry.provider.db.service.thrift.SentryPolicyServiceClientDefaultImpl;
-import org.apache.sentry.service.thrift.ServiceConstants.ClientConfig;
public final class SentryServiceClientFactory {
@@ -32,15 +31,9 @@ public final class SentryServiceClientFactory {
}
public static SentryPolicyServiceClient create(Configuration conf) throws Exception {
- boolean pooled = conf.getBoolean(ClientConfig.SENTRY_POOL_ENABLED, false);
- if (pooled) {
return (SentryPolicyServiceClient) Proxy
.newProxyInstance(SentryPolicyServiceClientDefaultImpl.class.getClassLoader(),
- SentryPolicyServiceClientDefaultImpl.class.getInterfaces(),
- new PoolClientInvocationHandler(conf));
- } else {
- return new SentryPolicyServiceClientDefaultImpl(conf);
- }
+ SentryPolicyServiceClientDefaultImpl.class.getInterfaces(),
+ new PoolClientInvocationHandler(conf));
}
-
}
http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceClientPoolFactory.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceClientPoolFactory.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceClientPoolFactory.java
index afea78a..0164fa6 100644
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceClientPoolFactory.java
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceClientPoolFactory.java
@@ -36,16 +36,21 @@ public class SentryServiceClientPoolFactory extends BasePooledObjectFactory<Sent
private static final Logger LOGGER = LoggerFactory.getLogger(SentryServiceClientPoolFactory.class);
- private Configuration conf;
+ private final String addr;
+ private final int port;
+ private final Configuration conf;
- public SentryServiceClientPoolFactory(Configuration conf) {
+ public SentryServiceClientPoolFactory(String addr, int port,
+ Configuration conf) {
+ this.addr = addr;
+ this.port = port;
this.conf = conf;
}
@Override
public SentryPolicyServiceClient create() throws Exception {
LOGGER.debug("Creating Sentry Service Client...");
- return new SentryPolicyServiceClientDefaultImpl(conf);
+ return new SentryPolicyServiceClientDefaultImpl(addr, port, conf);
}
@Override
http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/thrift/TestSentryServiceFailureCase.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/thrift/TestSentryServiceFailureCase.java b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/thrift/TestSentryServiceFailureCase.java
index 51bba31..d1ac447 100644
--- a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/thrift/TestSentryServiceFailureCase.java
+++ b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/thrift/TestSentryServiceFailureCase.java
@@ -58,10 +58,13 @@ public class TestSentryServiceFailureCase extends SentryServiceIntegrationBase {
public void testClientServerConnectionFailure() throws Exception {
try {
connectToSentryService();
+ String requestorUserName = ADMIN_USER;
+ client.listRoles(requestorUserName);
Assert.fail("Failed to receive Exception");
} catch(Exception e) {
LOGGER.info("Excepted exception", e);
- Throwable cause = e.getCause();
+ // peer callback exception is nested inside SentryUserException.
+ Throwable cause = e.getCause().getCause();
if (cause == null) {
throw e;
}
http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/thrift/TestSentryServiceWithInvalidMsgSize.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/thrift/TestSentryServiceWithInvalidMsgSize.java b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/thrift/TestSentryServiceWithInvalidMsgSize.java
index 15eab15..a4dd8a6 100644
--- a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/thrift/TestSentryServiceWithInvalidMsgSize.java
+++ b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/thrift/TestSentryServiceWithInvalidMsgSize.java
@@ -104,7 +104,7 @@ public class TestSentryServiceWithInvalidMsgSize extends SentryServiceIntegratio
client.grantServerPrivilege(ADMIN_USER, ROLE_NAME, "server", false);
} catch (SentryUserException e) {
exceptionThrown = true;
- Assert.assertTrue(e.getMessage().contains("org.apache.thrift.transport.TTransportException"));
+ Assert.assertTrue(e.getCause().getMessage().contains("org.apache.thrift.transport.TTransportException"));
} finally {
Assert.assertEquals(true, exceptionThrown);
}
http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestHMSFollower.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestHMSFollower.java b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestHMSFollower.java
new file mode 100644
index 0000000..d601b1e
--- /dev/null
+++ b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestHMSFollower.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sentry.service.thrift;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.api.*;
+import org.apache.hive.hcatalog.messaging.HCatEventMessage;
+import org.apache.sentry.binding.metastore.messaging.json.SentryJSONMessageFactory;
+import org.apache.sentry.provider.db.service.persistent.SentryStore;
+import org.apache.sentry.provider.db.service.thrift.TSentryAuthorizable;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+public class TestHMSFollower {
+ SentryJSONMessageFactory messageFactory = new SentryJSONMessageFactory();
+ SentryStore sentryStore = Mockito.mock(SentryStore.class);
+ final static String hiveInstance = "server2";
+
+ @Test
+ public void testCreateDatabase() throws Exception {
+ String dbName = "db1";
+
+ // Create notification events
+ NotificationEvent notificationEvent = new NotificationEvent(1, 0, HCatEventMessage.EventType.CREATE_DATABASE.toString(),
+ messageFactory.buildCreateDatabaseMessage(new Database(dbName, null, "hdfs://db1", null)).toString());
+ List<NotificationEvent> events = new ArrayList<>();
+ events.add(notificationEvent);
+
+ Configuration configuration = new Configuration();
+ HMSFollower hmsFollower = new HMSFollower(configuration, sentryStore, hiveInstance);
+ hmsFollower.processNotificationEvents(events);
+
+ TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance);
+ authorizable.setServer(hiveInstance);
+ authorizable.setDb("db1");
+
+ verify(sentryStore, times(1)).dropPrivilege(authorizable);
+ }
+ @Test
+ public void testDropDatabase() throws Exception {
+ String dbName = "db1";
+
+ // Create notification events
+ NotificationEvent notificationEvent = new NotificationEvent(1, 0, HCatEventMessage.EventType.DROP_DATABASE.toString(),
+ messageFactory.buildDropDatabaseMessage(new Database(dbName, null, "hdfs://db1", null)).toString());
+ List<NotificationEvent> events = new ArrayList<>();
+ events.add(notificationEvent);
+
+ Configuration configuration = new Configuration();
+ HMSFollower hmsFollower = new HMSFollower(configuration, sentryStore, hiveInstance);
+ hmsFollower.processNotificationEvents(events);
+
+ TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance);
+ authorizable.setServer(hiveInstance);
+ authorizable.setDb("db1");
+
+ verify(sentryStore, times(1)).dropPrivilege(authorizable);
+ }
+ @Test
+ public void testCreateTable() throws Exception {
+ String dbName = "db1";
+ String tableName = "table1";
+
+ // Create notification events
+ StorageDescriptor sd = new StorageDescriptor();
+ sd.setLocation("hdfs://db1.db/table1");
+ NotificationEvent notificationEvent = new NotificationEvent(1, 0, HCatEventMessage.EventType.CREATE_TABLE.toString(),
+ messageFactory.buildCreateTableMessage(new Table(tableName, dbName, null, 0, 0, 0, sd, null, null, null, null, null)).toString());
+ List<NotificationEvent> events = new ArrayList<>();
+ events.add(notificationEvent);
+
+ Configuration configuration = new Configuration();
+ HMSFollower hmsFollower = new HMSFollower(configuration, sentryStore, hiveInstance);
+ hmsFollower.processNotificationEvents(events);
+
+ TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance);
+ authorizable.setServer(hiveInstance);
+ authorizable.setDb("db1");
+ authorizable.setTable(tableName);
+
+ verify(sentryStore, times(1)).dropPrivilege(authorizable);
+ }
+ @Test
+ public void testDropTable() throws Exception {
+ String dbName = "db1";
+ String tableName = "table1";
+
+ // Create notification events
+ StorageDescriptor sd = new StorageDescriptor();
+ sd.setLocation("hdfs://db1.db/table1");
+ NotificationEvent notificationEvent = new NotificationEvent(1, 0, HCatEventMessage.EventType.DROP_TABLE.toString(),
+ messageFactory.buildDropTableMessage(new Table(tableName, dbName, null, 0, 0, 0, sd, null, null, null, null, null)).toString());
+ List<NotificationEvent> events = new ArrayList<>();
+ events.add(notificationEvent);
+
+ Configuration configuration = new Configuration();
+ HMSFollower hmsFollower = new HMSFollower(configuration, sentryStore, hiveInstance);
+ hmsFollower.processNotificationEvents(events);
+
+ TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance);
+ authorizable.setServer(hiveInstance);
+ authorizable.setDb("db1");
+ authorizable.setTable(tableName);
+
+ verify(sentryStore, times(1)).dropPrivilege(authorizable);
+ }
+ @Test
+ public void testRenameTable() throws Exception {
+ String dbName = "db1";
+ String tableName = "table1";
+
+ String newDbName = "db1";
+ String newTableName = "table2";
+
+ // Create notification events
+ StorageDescriptor sd = new StorageDescriptor();
+ sd.setLocation("hdfs://db1.db/table1");
+ NotificationEvent notificationEvent = new NotificationEvent(1, 0, HCatEventMessage.EventType.ALTER_TABLE.toString(),
+ messageFactory.buildAlterTableMessage(
+ new Table(tableName, dbName, null, 0, 0, 0, sd, null, null, null, null, null),
+ new Table(newTableName, newDbName, null, 0, 0, 0, sd, null, null, null, null, null)).toString());
+ notificationEvent.setDbName(newDbName);
+ notificationEvent.setTableName(newTableName);
+ List<NotificationEvent> events = new ArrayList<>();
+ events.add(notificationEvent);
+
+ Configuration configuration = new Configuration();
+ HMSFollower hmsFollower = new HMSFollower(configuration, sentryStore, hiveInstance);
+ hmsFollower.processNotificationEvents(events);
+
+ TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance);
+ authorizable.setServer(hiveInstance);
+ authorizable.setDb(dbName);
+ authorizable.setTable(tableName);
+
+ TSentryAuthorizable newAuthorizable = new TSentryAuthorizable(hiveInstance);
+ authorizable.setServer(hiveInstance);
+ newAuthorizable.setDb(newDbName);
+ newAuthorizable.setTable(newTableName);
+
+ verify(sentryStore, times(1)).renamePrivilege(authorizable, newAuthorizable);
+ }
+}
http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestPoolClientInvocationHandler.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestPoolClientInvocationHandler.java b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestPoolClientInvocationHandler.java
new file mode 100644
index 0000000..5b0e12b
--- /dev/null
+++ b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestPoolClientInvocationHandler.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.service.thrift;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TestPoolClientInvocationHandler {
+ private static final Logger LOGGER =
+ LoggerFactory.getLogger(TestPoolClientInvocationHandler.class);
+
+ private void expectParseHostPortStrings(String hostsAndPortsStr,
+ String[] expectedHosts, int[] expectedPorts) throws Exception {
+ boolean success = false;
+ String[] hostsAndPorts = hostsAndPortsStr.split(",");
+ String[] hosts = new String[hostsAndPorts.length];
+ int[] ports = new int[hostsAndPorts.length];
+ try {
+ PoolClientInvocationHandler.parseHostPortStrings(hostsAndPortsStr,
+ hostsAndPorts, hosts, ports, 8038);
+ success = true;
+ } finally {
+ if (!success) {
+ LOGGER.error("Caught exception while parsing hosts/ports string " +
+ hostsAndPortsStr);
+ }
+ }
+ Assert.assertArrayEquals("Got unexpected hosts results while " +
+ "parsing " + hostsAndPortsStr, expectedHosts, hosts);
+ Assert.assertArrayEquals("Got unexpected ports results while " +
+ "parsing " + hostsAndPortsStr, expectedPorts, ports);
+ }
+
+ @SuppressWarnings("PMD.AvoidUsingHardCodedIP")
+ @Test
+ public void testParseHostPortStrings() throws Exception {
+ expectParseHostPortStrings("foo", new String[] {"foo"}, new int[] {8038});
+ expectParseHostPortStrings("foo,bar",
+ new String[] {"foo", "bar"},
+ new int[] {8038, 8038});
+ expectParseHostPortStrings("foo:2020,bar:2021",
+ new String[] {"foo", "bar"},
+ new int[] {2020, 2021});
+ expectParseHostPortStrings("127.0.0.1:2020,127.1.0.1",
+ new String[] {"127.0.0.1", "127.1.0.1"},
+ new int[] {2020, 8038});
+ expectParseHostPortStrings("[2001:db8:85a3:8d3:1319:8a2e:370:7348]:433",
+ new String[] {"2001:db8:85a3:8d3:1319:8a2e:370:7348"},
+ new int[] {433});
+ }
+}
[2/2] sentry git commit: Sentry-1321: Implement HMSFollower in Sentry
service which reads the NotificationLog entries (Sravya Tirukkovalur,
Reviewed by: Hao Hao) (Append)
Posted by sr...@apache.org.
Sentry-1321: Implement HMSFollower in Sentry service which reads the NotificationLog entries (Sravya Tirukkovalur, Reviewed by: Hao Hao) (Append)
Change-Id: If6e3d4a754b71cc0ba0cec1e34a44e6d9816ea38
Project: http://git-wip-us.apache.org/repos/asf/sentry/repo
Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/908072d6
Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/908072d6
Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/908072d6
Branch: refs/heads/sentry-ha-redesign
Commit: 908072d66bbabd91491a7d8fe8d9fa558dbc9c46
Parents: 2d5ed99
Author: Sravya Tirukkovalur <sr...@apache.org>
Authored: Mon Aug 15 17:25:47 2016 -0700
Committer: Sravya Tirukkovalur <sr...@apache.org>
Committed: Mon Aug 15 17:25:47 2016 -0700
----------------------------------------------------------------------
sentry-binding/sentry-binding-hive-conf/pom.xml | 62 +++
.../sentry/binding/hive/conf/HiveAuthzConf.java | 269 ++++++++++++
.../sentry-binding-hive-follower/pom.xml | 71 ++++
.../json/JSONAlterPartitionMessage.java | 78 ++++
.../messaging/json/JSONAlterTableMessage.java | 68 +++
.../json/SentryJSONAddPartitionMessage.java | 49 +++
.../json/SentryJSONAlterPartitionMessage.java | 53 +++
.../json/SentryJSONAlterTableMessage.java | 50 +++
.../json/SentryJSONCreateDatabaseMessage.java | 44 ++
.../json/SentryJSONCreateTableMessage.java | 45 ++
.../json/SentryJSONDropDatabaseMessage.java | 44 ++
.../json/SentryJSONDropPartitionMessage.java | 49 +++
.../json/SentryJSONDropTableMessage.java | 45 ++
.../json/SentryJSONMessageDeserializer.java | 110 +++++
.../json/SentryJSONMessageFactory.java | 177 ++++++++
.../SentryInvalidHMSEventException.java | 31 ++
.../exception/SentryStandbyException.java | 4 +
.../SentryPolicyServiceClientDefaultImpl.java | 12 +-
.../sentry/service/thrift/HMSFollower.java | 411 +++++++++++++++++++
.../thrift/PoolClientInvocationHandler.java | 305 +++++++++++---
.../thrift/SentryServiceClientFactory.java | 11 +-
.../thrift/SentryServiceClientPoolFactory.java | 11 +-
.../thrift/TestSentryServiceFailureCase.java | 5 +-
.../TestSentryServiceWithInvalidMsgSize.java | 2 +-
.../sentry/service/thrift/TestHMSFollower.java | 165 ++++++++
.../thrift/TestPoolClientInvocationHandler.java | 69 ++++
26 files changed, 2173 insertions(+), 67 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-binding/sentry-binding-hive-conf/pom.xml
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive-conf/pom.xml b/sentry-binding/sentry-binding-hive-conf/pom.xml
new file mode 100644
index 0000000..2cae541
--- /dev/null
+++ b/sentry-binding/sentry-binding-hive-conf/pom.xml
@@ -0,0 +1,62 @@
+<?xml version="1.0"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.sentry</groupId>
+ <artifactId>sentry-binding</artifactId>
+ <version>1.8.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>sentry-binding-hive-conf</artifactId>
+ <name>Sentry Hive Binding Configuration</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.thrift</groupId>
+ <artifactId>libthrift</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.derby</groupId>
+ <artifactId>derby</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-exec</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-service</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
+
+</project>
http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-binding/sentry-binding-hive-conf/src/main/java/org/apache/sentry/binding/hive/conf/HiveAuthzConf.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive-conf/src/main/java/org/apache/sentry/binding/hive/conf/HiveAuthzConf.java b/sentry-binding/sentry-binding-hive-conf/src/main/java/org/apache/sentry/binding/hive/conf/HiveAuthzConf.java
new file mode 100644
index 0000000..4de755f
--- /dev/null
+++ b/sentry-binding/sentry-binding-hive-conf/src/main/java/org/apache/sentry/binding/hive/conf/HiveAuthzConf.java
@@ -0,0 +1,269 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sentry.binding.hive.conf;
+
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public class HiveAuthzConf extends Configuration {
+
+ /**
+ * Configuration key used in hive-site.xml to point at sentry-site.xml
+ */
+ public static final String HIVE_ACCESS_CONF_URL = "hive.access.conf.url";
+ public static final String HIVE_SENTRY_CONF_URL = "hive.sentry.conf.url";
+ public static final String HIVE_ACCESS_SUBJECT_NAME = "hive.access.subject.name";
+ public static final String HIVE_SENTRY_SUBJECT_NAME = "hive.sentry.subject.name";
+ public static final String HIVE_SENTRY_AUTH_ERRORS = "sentry.hive.authorization.errors";
+ public static final String HIVE_SENTRY_MOCK_COMPILATION = "sentry.hive.mock.compilation";
+ public static final String HIVE_SENTRY_MOCK_ERROR = "sentry.hive.mock.error";
+ public static final String HIVE_SENTRY_PRIVILEGE_ERROR_MESSAGE = "No valid privileges";
+ /**
+ * Property used to persist the role set in the session. This is not public for now.
+ */
+ public static final String SENTRY_ACTIVE_ROLE_SET = "hive.sentry.active.role.set";
+
+ public static final String HIVE_SENTRY_SECURITY_COMMAND_WHITELIST =
+ "hive.sentry.security.command.whitelist";
+ public static final String HIVE_SENTRY_SECURITY_COMMAND_WHITELIST_DEFAULT =
+ "set,reset,reload";
+
+ public static final String HIVE_SENTRY_SERDE_WHITELIST = "hive.sentry.serde.whitelist";
+ public static final String HIVE_SENTRY_SERDE_WHITELIST_DEFAULT = "org.apache.hadoop.hive.serde2";
+
+ // Disable the serde Uri privileges by default for backward compatibilities.
+ public static final String HIVE_SENTRY_SERDE_URI_PRIVILIEGES_ENABLED = "hive.sentry.turn.on.serde.uri.privileges";
+ public static final boolean HIVE_SENTRY_SERDE_URI_PRIVILIEGES_ENABLED_DEFAULT = false;
+
+ public static final String HIVE_UDF_WHITE_LIST =
+ "concat,substr,substring,space,repeat,ascii,lpad,rpad,size,round,floor,sqrt,ceil," +
+ "ceiling,rand,abs,pmod,ln,log2,sin,asin,cos,acos,log10,log,exp,power,pow,sign,pi," +
+ "degrees,radians,atan,tan,e,conv,bin,hex,unhex,base64,unbase64,encode,decode,upper," +
+ "lower,ucase,lcase,trim,ltrim,rtrim,length,reverse,field,find_in_set,initcap,like," +
+ "rlike,regexp,regexp_replace,regexp_extract,parse_url,nvl,split,str_to_map,translate" +
+ ",positive,negative,day,dayofmonth,month,year,hour,minute,second,from_unixtime," +
+ "to_date,weekofyear,last_day,date_add,date_sub,datediff,add_months,get_json_object," +
+ "xpath_string,xpath_boolean,xpath_number,xpath_double,xpath_float,xpath_long," +
+ "xpath_int,xpath_short,xpath,+,-,*,/,%,div,&,|,^,~,current_database,isnull," +
+ "isnotnull,if,in,and,or,=,==,<=>,!=,<>,<,<=,>,>=,not,!,between,ewah_bitmap_and," +
+ "ewah_bitmap_or,ewah_bitmap_empty,boolean,tinyint,smallint,int,bigint,float,double," +
+ "string,date,timestamp,binary,decimal,varchar,char,max,min,sum,count,avg,std,stddev," +
+ "stddev_pop,stddev_samp,variance,var_pop,var_samp,covar_pop,covar_samp,corr," +
+ "histogram_numeric,percentile_approx,collect_set,collect_list,ngrams," +
+ "context_ngrams,ewah_bitmap,compute_stats,percentile," +
+ "array,assert_true,map,struct,named_struct,create_union,case,when,hash,coalesce," +
+ "index,in_file,instr,locate,elt,concat_ws,sort_array," +
+ "array_contains,sentences,map_keys,map_values,format_number,printf,greatest,least," +
+ "from_utc_timestamp,to_utc_timestamp,unix_timestamp,to_unix_timestamp,explode," +
+ "inline,json_tuple,parse_url_tuple,posexplode,stack,lead,lag,row_number,rank," +
+ "dense_rank,percent_rank,cume_dist,ntile,first_value,last_value,noop,noopwithmap," +
+ "noopstreaming,noopwithmapstreaming,windowingtablefunction,matchpath";
+
+ public static final String HIVE_UDF_BLACK_LIST = "reflect,reflect2,java_method";
+
+ /**
+ * Config setting definitions
+ */
+ public static enum AuthzConfVars {
+ AUTHZ_PROVIDER("sentry.provider",
+ "org.apache.sentry.provider.common.HadoopGroupResourceAuthorizationProvider"),
+ AUTHZ_PROVIDER_RESOURCE("sentry.hive.provider.resource", ""),
+ AUTHZ_PROVIDER_BACKEND("sentry.hive.provider.backend", "org.apache.sentry.provider.file.SimpleFileProviderBackend"),
+ AUTHZ_POLICY_ENGINE("sentry.hive.policy.engine", "org.apache.sentry.policy.engine.common.CommonPolicyEngine"),
+ AUTHZ_POLICY_FILE_FORMATTER(
+ "sentry.hive.policy.file.formatter",
+ "org.apache.sentry.binding.hive.SentryIniPolicyFileFormatter"),
+ AUTHZ_SERVER_NAME("sentry.hive.server", ""),
+ AUTHZ_RESTRICT_DEFAULT_DB("sentry.hive.restrict.defaultDB", "false"),
+ SENTRY_TESTING_MODE("sentry.hive.testing.mode", "false"),
+ AUTHZ_ALLOW_HIVE_IMPERSONATION("sentry.hive.allow.hive.impersonation", "false"),
+ AUTHZ_ONFAILURE_HOOKS("sentry.hive.failure.hooks", ""),
+ AUTHZ_METASTORE_SERVICE_USERS("sentry.metastore.service.users", null),
+ AUTHZ_SYNC_ALTER_WITH_POLICY_STORE("sentry.hive.sync.alter", "true"),
+ AUTHZ_SYNC_CREATE_WITH_POLICY_STORE("sentry.hive.sync.create", "false"),
+ AUTHZ_SYNC_DROP_WITH_POLICY_STORE("sentry.hive.sync.drop", "true"),
+
+ AUTHZ_PROVIDER_DEPRECATED("hive.sentry.provider",
+ "org.apache.sentry.provider.file.ResourceAuthorizationProvider"),
+ AUTHZ_PROVIDER_RESOURCE_DEPRECATED("hive.sentry.provider.resource", ""),
+ AUTHZ_SERVER_NAME_DEPRECATED("hive.sentry.server", ""),
+ AUTHZ_RESTRICT_DEFAULT_DB_DEPRECATED("hive.sentry.restrict.defaultDB", "false"),
+ SENTRY_TESTING_MODE_DEPRECATED("hive.sentry.testing.mode", "false"),
+ AUTHZ_ALLOW_HIVE_IMPERSONATION_DEPRECATED("hive.sentry.allow.hive.impersonation", "false"),
+ AUTHZ_ONFAILURE_HOOKS_DEPRECATED("hive.sentry.failure.hooks", "");
+
+ private final String varName;
+ private final String defaultVal;
+
+ AuthzConfVars(String varName, String defaultVal) {
+ this.varName = varName;
+ this.defaultVal = defaultVal;
+ }
+
+ public String getVar() {
+ return varName;
+ }
+
+ public String getDefault() {
+ return defaultVal;
+ }
+
+ public static String getDefault(String varName) {
+ for (AuthzConfVars oneVar : AuthzConfVars.values()) {
+ if(oneVar.getVar().equalsIgnoreCase(varName)) {
+ return oneVar.getDefault();
+ }
+ }
+ return null;
+ }
+ }
+
+ // map of current property names - > deprecated property names.
+ // The binding layer code should work if the deprecated property names are provided,
+ // as long as the new property names aren't also provided. Since the binding code
+ // only calls the new property names, we require a map from current names to deprecated
+ // names in order to check if the deprecated name of a property was set.
+ private static final Map<String, AuthzConfVars> currentToDeprecatedProps =
+ new HashMap<String, AuthzConfVars>();
+ static {
+ currentToDeprecatedProps.put(AuthzConfVars.AUTHZ_PROVIDER.getVar(), AuthzConfVars.AUTHZ_PROVIDER_DEPRECATED);
+ currentToDeprecatedProps.put(AuthzConfVars.AUTHZ_PROVIDER_RESOURCE.getVar(), AuthzConfVars.AUTHZ_PROVIDER_RESOURCE_DEPRECATED);
+ currentToDeprecatedProps.put(AuthzConfVars.AUTHZ_SERVER_NAME.getVar(), AuthzConfVars.AUTHZ_SERVER_NAME_DEPRECATED);
+ currentToDeprecatedProps.put(AuthzConfVars.AUTHZ_RESTRICT_DEFAULT_DB.getVar(), AuthzConfVars.AUTHZ_RESTRICT_DEFAULT_DB_DEPRECATED);
+ currentToDeprecatedProps.put(AuthzConfVars.SENTRY_TESTING_MODE.getVar(), AuthzConfVars.SENTRY_TESTING_MODE_DEPRECATED);
+ currentToDeprecatedProps.put(AuthzConfVars.AUTHZ_ALLOW_HIVE_IMPERSONATION.getVar(), AuthzConfVars.AUTHZ_ALLOW_HIVE_IMPERSONATION_DEPRECATED);
+ currentToDeprecatedProps.put(AuthzConfVars.AUTHZ_ONFAILURE_HOOKS.getVar(), AuthzConfVars.AUTHZ_ONFAILURE_HOOKS_DEPRECATED);
+ };
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(HiveAuthzConf.class);
+ public static final String AUTHZ_SITE_FILE = "sentry-site.xml";
+ private final String hiveAuthzSiteFile;
+
+ public HiveAuthzConf(URL hiveAuthzSiteURL) {
+ super();
+ LOG.info("DefaultFS: " + super.get("fs.defaultFS"));
+ addResource(hiveAuthzSiteURL);
+ applySystemProperties();
+ LOG.info("DefaultFS: " + super.get("fs.defaultFS"));
+ this.hiveAuthzSiteFile = hiveAuthzSiteURL.toString();
+ }
+ /**
+ * Apply system properties to this object if the property name is defined in ConfVars
+ * and the value is non-null and not an empty string.
+ */
+ private void applySystemProperties() {
+ Map<String, String> systemProperties = getConfSystemProperties();
+ for (Entry<String, String> systemProperty : systemProperties.entrySet()) {
+ this.set(systemProperty.getKey(), systemProperty.getValue());
+ }
+ }
+
+ /**
+ * This method returns a mapping from config variable name to its value for all config variables
+ * which have been set using System properties
+ */
+ public static Map<String, String> getConfSystemProperties() {
+ Map<String, String> systemProperties = new HashMap<String, String>();
+
+ for (AuthzConfVars oneVar : AuthzConfVars.values()) {
+ String value = System.getProperty(oneVar.getVar());
+ if (value != null && value.length() > 0) {
+ systemProperties.put(oneVar.getVar(), value);
+ }
+ }
+ return systemProperties;
+ }
+
+ @Override
+ public String get(String varName) {
+ return get(varName, null);
+ }
+
+ @Override
+ public String get(String varName, String defaultVal) {
+ String retVal = super.get(varName);
+ if (retVal == null) {
+ // check if the deprecated value is set here
+ if (currentToDeprecatedProps.containsKey(varName)) {
+ retVal = super.get(currentToDeprecatedProps.get(varName).getVar());
+ }
+ if (retVal == null) {
+ retVal = AuthzConfVars.getDefault(varName);
+ } else {
+ LOG.warn("Using the deprecated config setting " + currentToDeprecatedProps.get(varName).getVar() +
+ " instead of " + varName);
+ }
+ }
+ if (retVal == null) {
+ retVal = defaultVal;
+ }
+ return retVal;
+ }
+
+ public String getHiveAuthzSiteFile() {
+ return hiveAuthzSiteFile;
+ }
+
+ /**
+ * Extract the authz config file path from given hive conf and load the authz config
+ * @param hiveConf
+ * @return
+ * @throws IllegalArgumentException
+ */
+ public static HiveAuthzConf getAuthzConf(HiveConf hiveConf)
+ throws IllegalArgumentException {
+ boolean depreicatedConfigFile = false;
+
+ String hiveAuthzConf = hiveConf.get(HiveAuthzConf.HIVE_SENTRY_CONF_URL);
+ if (hiveAuthzConf == null
+ || (hiveAuthzConf = hiveAuthzConf.trim()).isEmpty()) {
+ hiveAuthzConf = hiveConf.get(HiveAuthzConf.HIVE_ACCESS_CONF_URL);
+ depreicatedConfigFile = true;
+ }
+
+ if (hiveAuthzConf == null
+ || (hiveAuthzConf = hiveAuthzConf.trim()).isEmpty()) {
+ throw new IllegalArgumentException("Configuration key "
+ + HiveAuthzConf.HIVE_SENTRY_CONF_URL + " value '" + hiveAuthzConf
+ + "' is invalid.");
+ }
+
+ try {
+ return new HiveAuthzConf(new URL(hiveAuthzConf));
+ } catch (MalformedURLException e) {
+ if (depreicatedConfigFile) {
+ throw new IllegalArgumentException("Configuration key "
+ + HiveAuthzConf.HIVE_ACCESS_CONF_URL
+ + " specifies a malformed URL '" + hiveAuthzConf + "'", e);
+ } else {
+ throw new IllegalArgumentException("Configuration key "
+ + HiveAuthzConf.HIVE_SENTRY_CONF_URL
+ + " specifies a malformed URL '" + hiveAuthzConf + "'", e);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-binding/sentry-binding-hive-follower/pom.xml
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive-follower/pom.xml b/sentry-binding/sentry-binding-hive-follower/pom.xml
new file mode 100644
index 0000000..bcfb417
--- /dev/null
+++ b/sentry-binding/sentry-binding-hive-follower/pom.xml
@@ -0,0 +1,71 @@
+<?xml version="1.0"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.sentry</groupId>
+ <artifactId>sentry-binding</artifactId>
+ <version>1.8.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>sentry-binding-hive-follower</artifactId>
+ <name>Hive follower for Sentry</name>
+
+ <properties>
+ <datanucleus-api-jdo.version>3.2.6</datanucleus-api-jdo.version>
+ <datanucleus-core.version>3.2.12</datanucleus-core.version>
+ <datanucleus-rdbms.version>3.2.12</datanucleus-rdbms.version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.datanucleus</groupId>
+ <artifactId>datanucleus-core</artifactId>
+ <version>${datanucleus-core.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.datanucleus</groupId>
+ <artifactId>datanucleus-api-jdo</artifactId>
+ <version>${datanucleus-api-jdo.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.datanucleus</groupId>
+ <artifactId>datanucleus-rdbms</artifactId>
+ <version>${datanucleus-rdbms.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-exec</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive.hcatalog</groupId>
+ <artifactId>hive-hcatalog-server-extensions</artifactId>
+ <version>${hive.version}</version>
+ <scope>compile</scope>
+ </dependency>
+ </dependencies>
+
+</project>
http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterPartitionMessage.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterPartitionMessage.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterPartitionMessage.java
new file mode 100644
index 0000000..890186b
--- /dev/null
+++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterPartitionMessage.java
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.binding.metastore.messaging.json;
+
+import org.apache.hive.hcatalog.messaging.AlterPartitionMessage;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+import java.util.List;
+
+/*
+* This is only needed as corresponding class in Hive 1.1.0 does not have a default constructor
+ */
+public class JSONAlterPartitionMessage extends AlterPartitionMessage {
+ @JsonProperty
+ String server;
+ @JsonProperty
+ String servicePrincipal;
+ @JsonProperty
+ String db;
+ @JsonProperty
+ String table;
+ @JsonProperty
+ Long timestamp;
+ @JsonProperty
+ List<String> values;
+
+ public JSONAlterPartitionMessage() {}
+ public JSONAlterPartitionMessage(String server, String servicePrincipal, String db, String table, List<String> values, Long timestamp) {
+ this.server = server;
+ this.servicePrincipal = servicePrincipal;
+ this.db = db;
+ this.table = table;
+ this.timestamp = timestamp;
+ this.values = values;
+ this.checkValid();
+ }
+
+ public String getServer() {
+ return this.server;
+ }
+
+ public String getServicePrincipal() {
+ return this.servicePrincipal;
+ }
+
+ public String getDB() {
+ return this.db;
+ }
+
+ public Long getTimestamp() {
+ return this.timestamp;
+ }
+
+ public String getTable() {
+ return this.table;
+ }
+
+ public List<String> getValues() {
+ return this.values;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterTableMessage.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterTableMessage.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterTableMessage.java
new file mode 100644
index 0000000..76211c3
--- /dev/null
+++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterTableMessage.java
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.binding.metastore.messaging.json;
+
+import org.apache.hive.hcatalog.messaging.AlterTableMessage;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+/**
+ * This class is required as this class does not have a default contructor in Hive 1.1.0
+ */
+public class JSONAlterTableMessage extends AlterTableMessage {
+ @JsonProperty
+ String server;
+ @JsonProperty
+ String servicePrincipal;
+ @JsonProperty
+ String db;
+ @JsonProperty
+ String table;
+ @JsonProperty
+ Long timestamp;
+
+ public JSONAlterTableMessage() {}
+ public JSONAlterTableMessage(String server, String servicePrincipal, String db, String table, Long timestamp) {
+ this.server = server;
+ this.servicePrincipal = servicePrincipal;
+ this.db = db;
+ this.table = table;
+ this.timestamp = timestamp;
+ this.checkValid();
+ }
+
+ public String getServer() {
+ return this.server;
+ }
+
+ public String getServicePrincipal() {
+ return this.servicePrincipal;
+ }
+
+ public String getDB() {
+ return this.db;
+ }
+
+ public Long getTimestamp() {
+ return this.timestamp;
+ }
+
+ public String getTable() {
+ return this.table;
+ }
+}
http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAddPartitionMessage.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAddPartitionMessage.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAddPartitionMessage.java
new file mode 100644
index 0000000..c0c469c
--- /dev/null
+++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAddPartitionMessage.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.binding.metastore.messaging.json;
+
+import org.apache.hive.hcatalog.messaging.json.JSONAddPartitionMessage;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+import java.util.List;
+import java.util.Map;
+
+public class SentryJSONAddPartitionMessage extends JSONAddPartitionMessage {
+ @JsonProperty
+ List<String> locations;
+
+ public SentryJSONAddPartitionMessage() {
+ }
+
+ public SentryJSONAddPartitionMessage(String server, String servicePrincipal, String db, String table,
+ List<Map<String, String>> partitions, Long timestamp, List<String> locations) {
+ super(server, servicePrincipal, db, table, partitions, timestamp);
+ this.locations = locations;
+ }
+
+ public List<String> getLocations() {
+ return locations;
+ }
+
+ @Override
+ public String toString() {
+ return SentryJSONMessageDeserializer.serialize(this);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterPartitionMessage.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterPartitionMessage.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterPartitionMessage.java
new file mode 100644
index 0000000..99eb67a
--- /dev/null
+++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterPartitionMessage.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.binding.metastore.messaging.json;
+
+import org.codehaus.jackson.annotate.JsonProperty;
+
+import java.util.List;
+
+public class SentryJSONAlterPartitionMessage extends JSONAlterPartitionMessage{
+ @JsonProperty
+ String location;
+ @JsonProperty
+ String oldLocation;
+
+ public SentryJSONAlterPartitionMessage() {
+ }
+
+ public SentryJSONAlterPartitionMessage(String server, String servicePrincipal, String db, String table,
+ List<String> values, Long timestamp, String oldlocation, String newLocation) {
+ super(server, servicePrincipal, db, table, values, timestamp);
+ this.location = newLocation;
+ this.oldLocation = oldlocation;
+ }
+
+ public String getLocation() {
+ return location;
+ }
+
+ public String getOldLocation() {
+ return oldLocation;
+ }
+
+ @Override
+ public String toString() {
+ return SentryJSONMessageDeserializer.serialize(this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterTableMessage.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterTableMessage.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterTableMessage.java
new file mode 100644
index 0000000..6e59e25
--- /dev/null
+++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterTableMessage.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.binding.metastore.messaging.json;
+
+import org.codehaus.jackson.annotate.JsonProperty;
+
+public class SentryJSONAlterTableMessage extends JSONAlterTableMessage {
+ @JsonProperty
+ String location; //newLocation
+ @JsonProperty
+ String oldLocation;
+
+ public SentryJSONAlterTableMessage() {
+ }
+
+ public SentryJSONAlterTableMessage(String server, String servicePrincipal, String db, String table,
+ Long timestamp, String oldLocation, String location) {
+ super(server, servicePrincipal, db, table, timestamp);
+ this.location = location;
+ this.oldLocation = oldLocation;
+ }
+
+ public String getLocation() {
+ return location;
+ }
+ public String getOldLocation() {
+ return oldLocation;
+ }
+
+ @Override
+ public String toString() {
+ return SentryJSONMessageDeserializer.serialize(this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateDatabaseMessage.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateDatabaseMessage.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateDatabaseMessage.java
new file mode 100644
index 0000000..ba19cbe
--- /dev/null
+++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateDatabaseMessage.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.binding.metastore.messaging.json;
+
+import org.apache.hive.hcatalog.messaging.json.JSONCreateDatabaseMessage;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+public class SentryJSONCreateDatabaseMessage extends JSONCreateDatabaseMessage {
+ @JsonProperty
+ String location;
+
+ public SentryJSONCreateDatabaseMessage() {
+ }
+
+ public SentryJSONCreateDatabaseMessage(String server, String servicePrincipal, String db, Long timestamp, String location) {
+ super(server, servicePrincipal, db, timestamp);
+ this.location = location;
+ }
+
+ public String getLocation() {
+ return location;
+ }
+
+ @Override
+ public String toString() {
+ return SentryJSONMessageDeserializer.serialize(this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateTableMessage.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateTableMessage.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateTableMessage.java
new file mode 100644
index 0000000..57d11d2
--- /dev/null
+++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateTableMessage.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.binding.metastore.messaging.json;
+
+import org.apache.hive.hcatalog.messaging.json.JSONCreateTableMessage;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+public class SentryJSONCreateTableMessage extends JSONCreateTableMessage {
+ @JsonProperty
+ String location;
+
+ public SentryJSONCreateTableMessage() {
+ }
+
+ public SentryJSONCreateTableMessage(String server, String servicePrincipal, String db, String table, Long timestamp, String location) {
+ super(server, servicePrincipal, db, table, timestamp);
+ this.location = location;
+ }
+
+ public String getLocation() {
+ return location;
+ }
+
+ @Override
+ public String toString() {
+ return SentryJSONMessageDeserializer.serialize(this);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropDatabaseMessage.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropDatabaseMessage.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropDatabaseMessage.java
new file mode 100644
index 0000000..05f83f7
--- /dev/null
+++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropDatabaseMessage.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.binding.metastore.messaging.json;
+
+import org.apache.hive.hcatalog.messaging.json.JSONDropDatabaseMessage;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+public class SentryJSONDropDatabaseMessage extends JSONDropDatabaseMessage{
+ @JsonProperty
+ String location;
+
+ public SentryJSONDropDatabaseMessage() {
+ }
+
+ public SentryJSONDropDatabaseMessage(String server, String servicePrincipal, String db, Long timestamp, String location) {
+ super(server, servicePrincipal, db, timestamp);
+ this.location = location;
+ }
+
+ public String getLocation() {
+ return location;
+ }
+
+ @Override
+ public String toString() {
+ return SentryJSONMessageDeserializer.serialize(this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropPartitionMessage.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropPartitionMessage.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropPartitionMessage.java
new file mode 100644
index 0000000..2ab61f7
--- /dev/null
+++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropPartitionMessage.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.binding.metastore.messaging.json;
+
+import org.apache.hive.hcatalog.messaging.json.JSONDropPartitionMessage;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+import java.util.List;
+import java.util.Map;
+
+public class SentryJSONDropPartitionMessage extends JSONDropPartitionMessage {
+ @JsonProperty
+ String location;
+
+ public SentryJSONDropPartitionMessage() {
+ }
+
+ public SentryJSONDropPartitionMessage(String server, String servicePrincipal, String db, String table,
+ List<Map<String, String>> partitions, Long timestamp, String location) {
+ super(server, servicePrincipal, db, table, partitions, timestamp);
+ this.location = location;
+ }
+
+ public String getLocation() {
+ return location;
+ }
+
+ @Override
+ public String toString() {
+ return SentryJSONMessageDeserializer.serialize(this);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropTableMessage.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropTableMessage.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropTableMessage.java
new file mode 100644
index 0000000..7005776
--- /dev/null
+++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropTableMessage.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.binding.metastore.messaging.json;
+
+import org.apache.hive.hcatalog.messaging.json.JSONDropTableMessage;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+
+public class SentryJSONDropTableMessage extends JSONDropTableMessage {
+ @JsonProperty
+ String location;
+
+ public SentryJSONDropTableMessage() {
+ }
+
+ public SentryJSONDropTableMessage(String server, String servicePrincipal, String db, String table, Long timestamp, String location) {
+ super(server, servicePrincipal, db, table, timestamp);
+ this.location = location;
+ }
+
+ public String getLocation() {
+ return location;
+ }
+
+ @Override
+ public String toString() {
+ return SentryJSONMessageDeserializer.serialize(this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageDeserializer.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageDeserializer.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageDeserializer.java
new file mode 100644
index 0000000..b645c45
--- /dev/null
+++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageDeserializer.java
@@ -0,0 +1,110 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.binding.metastore.messaging.json;
+
+import org.apache.hive.hcatalog.messaging.*;
+import org.codehaus.jackson.map.DeserializationConfig;
+import org.codehaus.jackson.map.ObjectMapper;
+
+public class SentryJSONMessageDeserializer extends MessageDeserializer {
+ static ObjectMapper mapper = new ObjectMapper();
+
+ public SentryJSONMessageDeserializer() {
+ }
+
+ /**
+ * Method to de-serialize CreateDatabaseMessage instance.
+ */
+ public SentryJSONCreateDatabaseMessage getCreateDatabaseMessage(String messageBody) {
+ try {
+ return (SentryJSONCreateDatabaseMessage)mapper.readValue(messageBody, SentryJSONCreateDatabaseMessage.class);
+ } catch (Exception var3) {
+ throw new IllegalArgumentException("Could not construct SentryJSONCreateDatabaseMessage.", var3);
+ }
+ }
+
+ public SentryJSONDropDatabaseMessage getDropDatabaseMessage(String messageBody) {
+ try {
+ return (SentryJSONDropDatabaseMessage)mapper.readValue(messageBody, SentryJSONDropDatabaseMessage.class);
+ } catch (Exception var3) {
+ throw new IllegalArgumentException("Could not construct SentryJSONDropDatabaseMessage.", var3);
+ }
+ }
+
+ public SentryJSONCreateTableMessage getCreateTableMessage(String messageBody) {
+ try {
+ return (SentryJSONCreateTableMessage)mapper.readValue(messageBody, SentryJSONCreateTableMessage.class);
+ } catch (Exception var3) {
+ throw new IllegalArgumentException("Could not construct SentryJSONCreateTableMessage.", var3);
+ }
+ }
+
+ public SentryJSONAlterTableMessage getAlterTableMessage(String messageBody) {
+ try {
+ return (SentryJSONAlterTableMessage)mapper.readValue(messageBody, SentryJSONAlterTableMessage.class);
+ } catch (Exception var3) {
+ throw new IllegalArgumentException("Could not construct SentryJSONAlterTableMessage.", var3);
+ }
+ }
+
+ public SentryJSONDropTableMessage getDropTableMessage(String messageBody) {
+ try {
+ return (SentryJSONDropTableMessage)mapper.readValue(messageBody, SentryJSONDropTableMessage.class);
+ } catch (Exception var3) {
+ throw new IllegalArgumentException("Could not construct SentryJSONDropTableMessage.", var3);
+ }
+ }
+
+ public SentryJSONAddPartitionMessage getAddPartitionMessage(String messageBody) {
+ try {
+ return (SentryJSONAddPartitionMessage)mapper.readValue(messageBody, SentryJSONAddPartitionMessage.class);
+ } catch (Exception var3) {
+ throw new IllegalArgumentException("Could not construct SentryJSONAddPartitionMessage.", var3);
+ }
+ }
+
+ public SentryJSONAlterPartitionMessage getAlterPartitionMessage(String messageBody) {
+ try {
+ return (SentryJSONAlterPartitionMessage)mapper.readValue(messageBody, SentryJSONAlterPartitionMessage.class);
+ } catch (Exception var3) {
+ throw new IllegalArgumentException("Could not construct SentryJSONAlterPartitionMessage.", var3);
+ }
+ }
+
+ public SentryJSONDropPartitionMessage getDropPartitionMessage(String messageBody) {
+ try {
+ return (SentryJSONDropPartitionMessage)mapper.readValue(messageBody, SentryJSONDropPartitionMessage.class);
+ } catch (Exception var3) {
+ throw new IllegalArgumentException("Could not construct SentryJSONDropPartitionMessage.", var3);
+ }
+ }
+
+ static {
+ mapper.configure(DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+ }
+
+ public static String serialize(Object object) {
+ try {
+ return mapper.writeValueAsString(object);
+ }
+ catch (Exception exception) {
+ throw new IllegalArgumentException("Could not serialize: ", exception);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageFactory.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageFactory.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageFactory.java
new file mode 100644
index 0000000..00e7db8
--- /dev/null
+++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageFactory.java
@@ -0,0 +1,177 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sentry.binding.metastore.messaging.json;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.common.classification.InterfaceAudience;
+import org.apache.hadoop.hive.common.classification.InterfaceStability;
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy;
+import org.apache.hive.hcatalog.messaging.*;
+
+import java.util.*;
+
+public class SentryJSONMessageFactory extends MessageFactory {
+ private static final Log LOG = LogFactory.getLog(SentryJSONMessageFactory.class.getName());
+ private static SentryJSONMessageDeserializer deserializer = new SentryJSONMessageDeserializer();
+ public SentryJSONMessageFactory() {
+ LOG.info("Using SentryJSONMessageFactory for building Notification log messages ");
+
+ }
+ public MessageDeserializer getDeserializer() {
+ return deserializer;
+ }
+
+ public String getVersion() {
+ return "0.1";
+ }
+
+ public String getMessageFormat() {
+ return "json";
+ }
+
+ public SentryJSONCreateDatabaseMessage buildCreateDatabaseMessage(Database db) {
+ return new SentryJSONCreateDatabaseMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, db.getName(),
+ Long.valueOf(this.now()), db.getLocationUri());
+ }
+ public SentryJSONDropDatabaseMessage buildDropDatabaseMessage(Database db) {
+ return new SentryJSONDropDatabaseMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, db.getName(),
+ Long.valueOf(this.now()), db.getLocationUri());
+ }
+
+ public SentryJSONCreateTableMessage buildCreateTableMessage(Table table) {
+ return new SentryJSONCreateTableMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, table.getDbName(),
+ table.getTableName(), Long.valueOf(this.now()), table.getSd().getLocation());
+ }
+
+ public SentryJSONAlterTableMessage buildAlterTableMessage(Table before, Table after) {
+ return new SentryJSONAlterTableMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, before.getDbName(),
+ before.getTableName(), Long.valueOf(this.now()), before.getSd().getLocation(), after.getSd().getLocation());
+ }
+
+ public SentryJSONDropTableMessage buildDropTableMessage(Table table) {
+ return new SentryJSONDropTableMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, table.getDbName(),
+ table.getTableName(), Long.valueOf(this.now()), table.getSd().getLocation());
+ }
+
+ public SentryJSONAddPartitionMessage buildAddPartitionMessage(Table table, List<Partition> partitions) {
+ return new SentryJSONAddPartitionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, table.getDbName(),
+ table.getTableName(), getPartitionKeyValues(table, partitions), Long.valueOf(this.now()),
+ getPartitionLocations(partitions));
+ }
+
+ private List<String> getPartitionLocations(List<Partition> partitions) {
+ List<String> paths = new ArrayList<String>();
+ for(Partition partition:partitions) {
+ paths.add(partition.getSd().getLocation());
+ }
+ return paths;
+ }
+
+ //TODO: Not sure what is this used for. Need to investigate
+ private List<String> getPartitionLocations(PartitionSpecProxy partitionSpec) {
+ Iterator<Partition> iterator = partitionSpec.getPartitionIterator();
+ List<String> locations = new ArrayList<String>();
+ while(iterator.hasNext()) {
+ locations.add(iterator.next().getSd().getLocation());
+ }
+ return locations;
+ }
+
+ @InterfaceAudience.LimitedPrivate({"Hive"})
+ @InterfaceStability.Evolving
+ public SentryJSONAddPartitionMessage buildAddPartitionMessage(Table table, PartitionSpecProxy partitionSpec) {
+ return new SentryJSONAddPartitionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, table.getDbName(),
+ table.getTableName(), getPartitionKeyValues(table, partitionSpec), Long.valueOf(this.now()),
+ getPartitionLocations(partitionSpec));
+ }
+
+ public SentryJSONAlterPartitionMessage buildAlterPartitionMessage(Partition before, Partition after) {
+ /*
+ f (partitionEvent.getOldPartition() != null) {
+ oldLoc = partitionEvent.getOldPartition().getSd().getLocation();
+ }
+ if (partitionEvent.getNewPartition() != null) {
+ newLoc = partitionEvent.getNewPartition().getSd().getLocation();
+ }
+
+ if ((oldLoc != null) && (newLoc != null) && (!oldLoc.equals(newLoc))) {
+ String authzObj =
+ partitionEvent.getOldPartition().getDbName() + "."
+ + partitionEvent.getOldPartition().getTableName();
+ for (SentryMetastoreListenerPlugin plugin : sentryPlugins) {
+ plugin.renameAuthzObject(authzObj, oldLoc,
+ authzObj, newLoc);
+ }
+ }
+ * */
+ return new SentryJSONAlterPartitionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, before.getDbName(),
+ before.getTableName(), before.getValues(), Long.valueOf(this.now()), before.getSd().getLocation(),
+ after.getSd().getLocation());
+ }
+
+ public SentryJSONDropPartitionMessage buildDropPartitionMessage(Table table, Partition partition) {
+ return new SentryJSONDropPartitionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, partition.getDbName(),
+ partition.getTableName(), Arrays.asList(getPartitionKeyValues(table, partition)),
+ Long.valueOf(this.now()), partition.getSd().getLocation());
+ }
+
+ private static Map<String, String> getPartitionKeyValues(Table table, Partition partition) {
+ LinkedHashMap partitionKeys = new LinkedHashMap();
+
+ for(int i = 0; i < table.getPartitionKeysSize(); ++i) {
+ partitionKeys.put(((FieldSchema)table.getPartitionKeys().get(i)).getName(), partition.getValues().get(i));
+ }
+
+ return partitionKeys;
+ }
+
+ private static List<Map<String, String>> getPartitionKeyValues(Table table, List<Partition> partitions) {
+ ArrayList partitionList = new ArrayList(partitions.size());
+ Iterator i$ = partitions.iterator();
+
+ while(i$.hasNext()) {
+ Partition partition = (Partition)i$.next();
+ partitionList.add(getPartitionKeyValues(table, partition));
+ }
+
+ return partitionList;
+ }
+
+ @InterfaceAudience.LimitedPrivate({"Hive"})
+ @InterfaceStability.Evolving
+ private static List<Map<String, String>> getPartitionKeyValues(Table table, PartitionSpecProxy partitionSpec) {
+ ArrayList partitionList = new ArrayList();
+ PartitionSpecProxy.PartitionIterator iterator = partitionSpec.getPartitionIterator();
+
+ while(iterator.hasNext()) {
+ Partition partition = (Partition)iterator.next();
+ partitionList.add(getPartitionKeyValues(table, partition));
+ }
+
+ return partitionList;
+ }
+ //This is private in parent class
+ private long now() {
+ return System.currentTimeMillis() / 1000L;
+ }
+}
http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/core/common/exception/SentryInvalidHMSEventException.java
----------------------------------------------------------------------
diff --git a/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/core/common/exception/SentryInvalidHMSEventException.java b/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/core/common/exception/SentryInvalidHMSEventException.java
new file mode 100644
index 0000000..50fb59c
--- /dev/null
+++ b/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/core/common/exception/SentryInvalidHMSEventException.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sentry.core.common.exception;
+
+public class SentryInvalidHMSEventException extends SentryUserException {
+ private static final long serialVersionUID = 29620806553835L;
+ public SentryInvalidHMSEventException(String msg) {
+ super(msg);
+ }
+ public SentryInvalidHMSEventException(String msg, String reason) {
+ super(msg, reason);
+ }
+ public SentryInvalidHMSEventException(String msg, Throwable cause) {
+ super(msg, cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/core/common/exception/SentryStandbyException.java
----------------------------------------------------------------------
diff --git a/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/core/common/exception/SentryStandbyException.java b/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/core/common/exception/SentryStandbyException.java
index b2df699..da6cfce 100644
--- a/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/core/common/exception/SentryStandbyException.java
+++ b/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/core/common/exception/SentryStandbyException.java
@@ -34,4 +34,8 @@ public class SentryStandbyException extends SentryUserException {
public SentryStandbyException(String msg, String reason) {
super(msg, reason);
}
+
+ public SentryStandbyException(String msg, Throwable t) {
+ super(msg, t);
+ }
}
http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryPolicyServiceClientDefaultImpl.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryPolicyServiceClientDefaultImpl.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryPolicyServiceClientDefaultImpl.java
index 1039e6e..4f42a51 100644
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryPolicyServiceClientDefaultImpl.java
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryPolicyServiceClientDefaultImpl.java
@@ -132,12 +132,18 @@ public class SentryPolicyServiceClientDefaultImpl implements SentryPolicyService
}
public SentryPolicyServiceClientDefaultImpl(Configuration conf) throws IOException {
+ this(Preconditions.checkNotNull(conf.get(ClientConfig.SERVER_RPC_ADDRESS), "Config key "
+ + ClientConfig.SERVER_RPC_ADDRESS + " is required"), conf.getInt(
+ ClientConfig.SERVER_RPC_PORT, ClientConfig.SERVER_RPC_PORT_DEFAULT), conf);
+ }
+
+ public SentryPolicyServiceClientDefaultImpl(String addr, int port,
+ Configuration conf) throws IOException {
this.conf = conf;
Preconditions.checkNotNull(this.conf, "Configuration object cannot be null");
this.serverAddress = NetUtils.createSocketAddr(Preconditions.checkNotNull(
- conf.get(ClientConfig.SERVER_RPC_ADDRESS), "Config key "
- + ClientConfig.SERVER_RPC_ADDRESS + " is required"), conf.getInt(
- ClientConfig.SERVER_RPC_PORT, ClientConfig.SERVER_RPC_PORT_DEFAULT));
+ addr, "Config key " + ClientConfig.SERVER_RPC_ADDRESS
+ + " is required"), port);
this.connectionTimeout = conf.getInt(ClientConfig.SERVER_RPC_CONN_TIMEOUT,
ClientConfig.SERVER_RPC_CONN_TIMEOUT_DEFAULT);
kerberos = ServerConfig.SECURITY_MODE_KERBEROS.equalsIgnoreCase(
http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java
new file mode 100644
index 0000000..4430471
--- /dev/null
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java
@@ -0,0 +1,411 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sentry.service.thrift;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.NotificationEvent;
+import org.apache.hadoop.hive.metastore.api.NotificationEventResponse;
+import org.apache.hadoop.security.SaslRpcServer;
+import org.apache.hive.hcatalog.messaging.HCatEventMessage;
+import org.apache.sentry.binding.hive.conf.HiveAuthzConf;
+import org.apache.sentry.core.common.exception.*;
+import org.apache.sentry.provider.db.service.persistent.SentryStore;
+import org.apache.sentry.provider.db.service.thrift.TSentryAuthorizable;
+import org.apache.thrift.TException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.sentry.binding.metastore.messaging.json.*;
+
+import javax.security.auth.Subject;
+import javax.security.auth.login.LoginException;
+import java.io.File;
+import java.io.IOException;
+import java.security.PrivilegedActionException;
+import java.security.PrivilegedExceptionAction;
+import java.util.List;
+
+import static org.apache.sentry.binding.hive.conf.HiveAuthzConf.AuthzConfVars.AUTHZ_SYNC_CREATE_WITH_POLICY_STORE;
+import static org.apache.sentry.binding.hive.conf.HiveAuthzConf.AuthzConfVars.AUTHZ_SYNC_DROP_WITH_POLICY_STORE;
+
+/*
+HMSFollower is the thread which follows the Hive MetaStore state changes from Sentry.
+It gets the full update and notification logs from HMS and applies it to
+update permissions stored in Sentry using SentryStore and also update the <obj,path> state
+stored for HDFS- Sentry sync.
+ */
+@SuppressWarnings("PMD")
+public class HMSFollower implements Runnable {
+ private static final Logger LOGGER = LoggerFactory.getLogger(HMSFollower.class);
+
+ private long currentEventID;
+ private static boolean connectedToHMS = false;
+ private HiveMetaStoreClient client;
+ private SentryKerberosContext kerberosContext;
+ private Configuration authzConf;
+ private boolean kerberos;
+ private SentryStore sentryStore;
+ private String hiveInstance;
+ final static int maxRetriesForLogin = 3;
+ final static int maxRetriesForConnection = 3;
+
+ HMSFollower(Configuration conf) throws SentryNoSuchObjectException,
+ SentryAccessDeniedException, SentrySiteConfigurationException, IOException { //TODO: Handle any possible exceptions or throw specific exceptions
+ LOGGER.info("HMSFollower is being initialized");
+ authzConf = conf;
+ try {
+ sentryStore = new SentryStore(authzConf);
+ } catch (SentryStandbyException e) {
+ //TODO: Do not start HMSFollower if standby
+ }
+ //TODO: Initialize currentEventID from Sentry db
+ currentEventID = 0;
+ }
+
+ @VisibleForTesting
+ HMSFollower(Configuration conf, SentryStore sentryStore, String hiveInstance) {
+ this.authzConf = conf;
+ this.sentryStore = sentryStore;
+ this.hiveInstance = hiveInstance;
+ }
+
+ @VisibleForTesting
+ public static boolean isConnectedToHMS() {
+ return connectedToHMS;
+ }
+
+ /*
+ Returns HMS Client if successful, returns null if HMS is not ready yet to take connections
+ Throws @LoginException if Kerberos context creation failed using Sentry's kerberos credentials
+ Throws @MetaException if there was a problem on creating an HMSClient
+ */
+ private HiveMetaStoreClient getMetaStoreClient(Configuration conf)
+ throws LoginException, MetaException {
+ if(client != null) {
+ return client;
+ }
+ // Seems like HMS client creation although seems successful,
+ // it actually connects to an invalid HMS instance.
+ // So it seems like it is necessary to wait until we make sure metastore config is properly loaded.
+ boolean loadedHiveConf = HiveConf.isLoadMetastoreConfig();
+ if(!loadedHiveConf) {
+ return null;
+ }
+ final HiveConf hiveConf = new HiveConf();
+ hiveInstance = hiveConf.get(HiveAuthzConf.AuthzConfVars.AUTHZ_SERVER_NAME.getVar());
+
+ String principal, keytab;
+
+ //TODO: Is this the right(standard) way to create a HMS client? HiveMetastoreClientFactoryImpl?
+ //TODO: Check if HMS is using kerberos instead of relying on Sentry conf
+ //TODO: Handle TGT renewals
+ kerberos = ServiceConstants.ServerConfig.SECURITY_MODE_KERBEROS.equalsIgnoreCase(
+ conf.get(ServiceConstants.ServerConfig.SECURITY_MODE, ServiceConstants.ServerConfig.SECURITY_MODE_KERBEROS).trim());
+ if (kerberos) {
+ LOGGER.info("Making a kerberos connection to HMS");
+ //TODO: Is this needed? Use Hadoop libraries to translate the _HOST placeholder with actual hostname
+ //Validate principal
+ principal = Preconditions.checkNotNull(ServiceConstants.ServerConfig.PRINCIPAL,
+ ServiceConstants.ServerConfig.PRINCIPAL + " is required");
+ LOGGER.info("Using kerberos principal: " + principal);
+ final String[] principalParts = SaslRpcServer.splitKerberosName(principal);
+ Preconditions.checkArgument(principalParts.length == 3,
+ "Kerberos principal should have 3 parts: " + principal);
+
+ keytab = Preconditions.checkNotNull(conf.get(ServiceConstants.ServerConfig.KEY_TAB),
+ ServiceConstants.ServerConfig.KEY_TAB + " is required");
+ File keytabFile = new File(keytab);
+ Preconditions.checkState(keytabFile.isFile() && keytabFile.canRead(),
+ "Keytab " + keytab + " does not exist or is not readable.");
+ boolean establishedKerberosContext = false;
+ int attempt = 1;
+ while(establishedKerberosContext) {
+ try {
+ kerberosContext = new SentryKerberosContext(principal, keytab, true);
+ establishedKerberosContext = true;
+ LOGGER.info("Established kerberos context, will now connect to HMS");
+ } catch (LoginException e) {
+ //Kerberos login failed
+ if( attempt > maxRetriesForLogin ) {
+ throw e;
+ }
+ attempt++;
+ }
+ }
+ boolean establishedConnection = false;
+ attempt = 1;
+ while(establishedConnection) {
+ try {
+ client = Subject.doAs(kerberosContext.getSubject(), new PrivilegedExceptionAction<HiveMetaStoreClient>() {
+ @Override
+ public HiveMetaStoreClient run() throws Exception {
+ return new HiveMetaStoreClient(hiveConf);
+ }
+ });
+ LOGGER.info("Secure connection established with HMS");
+ } catch (PrivilegedActionException e) {
+ if( attempt > maxRetriesForConnection ) {
+ //We should just retry as it is possible that HMS is not ready yet to receive requests
+ //TODO: How do we differentiate between kerberos problem versus HMS not being up?
+ LOGGER.error("Cannot connect to HMS", e);
+ }
+ attempt++;
+ }
+ }
+ } else {
+ //This is only for testing purposes. Sentry strongly recommends strong authentication
+ client = new HiveMetaStoreClient(hiveConf);
+ LOGGER.info("Non secure connection established with HMS");
+ }
+ return client;
+ }
+
+ public void run() {
+ if( client == null ) {
+ try {
+ client = getMetaStoreClient(authzConf);
+ if (client == null) {
+ //TODO: Do we want to throw an exception after a certain timeout?
+ return;
+ } else {
+ connectedToHMS = true;
+ LOGGER.info("HMSFollower of Sentry successfully connected to HMS");
+ }
+ } catch (Exception e) {
+
+ }
+ }
+ if (needFullUpdate()) {
+ //TODO: Handle
+ } else {
+ try {
+ NotificationEventResponse response = client.getNextNotification(currentEventID, Integer.MAX_VALUE, null);
+ if (response.isSetEvents()) {
+ LOGGER.info(String.format("CurrentEventID = %s. Processing %s events",
+ currentEventID, response.getEvents().size()));
+ processNotificationEvents(response.getEvents());
+ }
+ } catch (TException e) {
+ LOGGER.error("ThriftException occured fetching Notification entries, will try");
+ e.printStackTrace();
+ } catch (SentryInvalidInputException|SentryInvalidHMSEventException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ private boolean needFullUpdate() {
+ //TODO Implement
+ return false;
+ }
+
+ private boolean syncWithPolicyStore(HiveAuthzConf.AuthzConfVars syncConfVar) {
+ return "true"
+ .equalsIgnoreCase((authzConf.get(syncConfVar.getVar(), "true")));
+ }
+
+ /*
+ Throws SentryInvalidHMSEventException if Notification event contains insufficient information
+ */
+
+ void processNotificationEvents(List<NotificationEvent> events) throws
+ SentryInvalidHMSEventException, SentryInvalidInputException {
+ SentryJSONMessageDeserializer deserializer = new SentryJSONMessageDeserializer();
+
+ for (NotificationEvent event : events) {
+ String dbName, tableName, oldLocation, newLocation, location;
+ switch (HCatEventMessage.EventType.valueOf(event.getEventType())) {
+ case CREATE_DATABASE:
+ SentryJSONCreateDatabaseMessage message = deserializer.getCreateDatabaseMessage(event.getMessage());
+ dbName = message.getDB();
+
+ location = message.getLocation();
+ if (dbName == null || location == null) {
+ throw new SentryInvalidHMSEventException(String.format("Create database event has incomplete information. " +
+ "dbName = %s location = %s", dbName, location));
+ }
+ if (syncWithPolicyStore(AUTHZ_SYNC_CREATE_WITH_POLICY_STORE)) {
+ try {
+ dropSentryDbPrivileges(dbName);
+ } catch (SentryNoSuchObjectException e) {
+ LOGGER.info("Drop Sentry privilege ignored as there are no privileges on the database: %s", dbName);
+ } catch (SentryInvalidInputException e) {
+ throw new SentryInvalidInputException("Could not process Create database event. Event: " + event.toString(), e);
+ }
+ }
+ //TODO: HDFSPlugin.addPath(dbName, location)
+ break;
+ case DROP_DATABASE:
+ SentryJSONDropDatabaseMessage dropDatabaseMessage = deserializer.getDropDatabaseMessage(event.getMessage());
+ dbName = dropDatabaseMessage.getDB();
+ if (dbName == null) {
+ throw new SentryInvalidHMSEventException(String.format("Drop database event has incomplete information. " +
+ "dbName = %s", dbName));
+ }
+ if (syncWithPolicyStore(AUTHZ_SYNC_DROP_WITH_POLICY_STORE)) {
+ try {
+ dropSentryDbPrivileges(dbName);
+ } catch (SentryNoSuchObjectException e) {
+ LOGGER.info("Drop Sentry privilege ignored as there are no privileges on the database: %s", dbName);
+ } catch (SentryInvalidInputException e) {
+ throw new SentryInvalidInputException("Could not process Drop database event. Event: " + event.toString(), e);
+ }
+ }
+ //TODO: HDFSPlugin.deletePath(dbName, location)
+ break;
+ case CREATE_TABLE:
+ SentryJSONCreateTableMessage createTableMessage = deserializer.getCreateTableMessage(event.getMessage());
+ dbName = createTableMessage.getDB();
+ tableName = createTableMessage.getTable();
+ location = createTableMessage.getLocation();
+ if (dbName == null || tableName == null || location == null) {
+ throw new SentryInvalidHMSEventException(String.format("Create table event has incomplete information. " +
+ "dbName = %s, tableName = %s, location = %s", dbName, tableName, location));
+ }
+ if (syncWithPolicyStore(AUTHZ_SYNC_CREATE_WITH_POLICY_STORE)) {
+ try {
+ dropSentryTablePrivileges(dbName, tableName);
+ } catch (SentryNoSuchObjectException e) {
+ LOGGER.info("Drop Sentry privilege ignored as there are no privileges on the table: %s.%s", dbName, tableName);
+ } catch (SentryInvalidInputException e) {
+ throw new SentryInvalidInputException("Could not process Create table event. Event: " + event.toString(), e);
+ }
+ }
+ //TODO: HDFSPlugin.deletePath(dbName, location)
+ break;
+ case DROP_TABLE:
+ SentryJSONDropTableMessage dropTableMessage = deserializer.getDropTableMessage(event.getMessage());
+ dbName = dropTableMessage.getDB();
+ tableName = dropTableMessage.getTable();
+ if (dbName == null || tableName == null) {
+ throw new SentryInvalidHMSEventException(String.format("Drop table event has incomplete information. " +
+ "dbName = %s, tableName = %s", dbName, tableName));
+ }
+ if (syncWithPolicyStore(AUTHZ_SYNC_DROP_WITH_POLICY_STORE)) {
+ try{
+ dropSentryTablePrivileges(dbName, tableName);
+ } catch (SentryNoSuchObjectException e) {
+ LOGGER.info("Drop Sentry privilege ignored as there are no privileges on the table: %s.%s", dbName, tableName);
+ } catch (SentryInvalidInputException e) {
+ throw new SentryInvalidInputException("Could not process Drop table event. Event: " + event.toString(), e);
+ }
+ }
+ //TODO: HDFSPlugin.deletePath(dbName, location)
+ break;
+ case ALTER_TABLE:
+ SentryJSONAlterTableMessage alterTableMessage = deserializer.getAlterTableMessage(event.getMessage());
+
+ String oldDbName = alterTableMessage.getDB();
+ String oldTableName = alterTableMessage.getTable();
+ String newDbName = event.getDbName();
+ String newTableName = event.getTableName();
+ oldLocation = alterTableMessage.getOldLocation();
+ newLocation = alterTableMessage.getLocation();
+
+ if (oldDbName == null || oldTableName == null || newDbName == null || newTableName == null ||
+ oldLocation == null || newLocation == null) {
+ throw new SentryInvalidHMSEventException(String.format("Alter table event has incomplete information. " +
+ "oldDbName = %s, oldTableName = %s, oldLocation = %s, newDbName = %s, newTableName = %s, newLocation = %s",
+ oldDbName, oldTableName, oldLocation, newDbName, newTableName, newLocation));
+ }
+
+ if(!newDbName.equalsIgnoreCase(oldDbName) || !oldTableName.equalsIgnoreCase(newTableName)) { // Name has changed
+ if(!oldLocation.equals(newLocation)) { // Location has changed
+
+ //Name and path has changed
+ // - Alter table rename for managed table
+ //TODO: Handle HDFS plugin
+
+ } else {
+ //Only name has changed
+ // - Alter table rename for an external table
+ //TODO: Handle HDFS plugin
+
+ }
+ try {
+ renamePrivileges(oldDbName, oldTableName, newDbName, newTableName);
+ } catch (SentryNoSuchObjectException e) {
+ LOGGER.info("Rename Sentry privilege ignored as there are no privileges on the table: %s.%s", oldDbName, oldTableName);
+ } catch (SentryInvalidInputException e) {
+ throw new SentryInvalidInputException("Could not process Alter table event. Event: " + event.toString(), e);
+ } catch (SentryStandbyException e) {
+ LOGGER.error("Seems like this process became a standby. " +
+ "Ignoring the failure. The new leader will reprocess this notification event");
+ }
+ } else if(!oldLocation.equals(newLocation)) { // Only Location has changed{
+ //- Alter table set location
+ //TODO: Handle HDFS plugin
+ } else {
+ LOGGER.info(String.format("Alter table notification ignored as neither name nor location has changed: " +
+ "oldDbName = %s, oldTableName = %s, oldLocation = %s, newDbName = %s, newTableName = %s, newLocation = %s",
+ oldDbName, oldTableName, oldLocation, newDbName, newTableName, newLocation));
+ }
+ //TODO: Write test cases for all these cases
+ break;
+ case ADD_PARTITION:
+ case DROP_PARTITION:
+ case ALTER_PARTITION:
+ //TODO: Handle HDFS plugin
+ break;
+ }
+ currentEventID = event.getEventId();
+ }
+ }
+
+ private void dropSentryDbPrivileges(String dbName) throws SentryNoSuchObjectException, SentryInvalidInputException {
+ TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance);
+ authorizable.setDb(dbName);
+ try {
+ sentryStore.dropPrivilege(authorizable);
+ } catch (SentryStandbyException e) {
+ LOGGER.error("Seems like this process became a standby. " +
+ "Ignoring the failure. The new leader will reprocess this notification event");
+ }
+ }
+ private void dropSentryTablePrivileges(String dbName, String tableName) throws SentryNoSuchObjectException,
+ SentryInvalidInputException {
+ TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance);
+ authorizable.setDb(dbName);
+ authorizable.setTable(tableName);
+ try {
+ sentryStore.dropPrivilege(authorizable);
+ } catch (SentryStandbyException e) {
+ LOGGER.error("Seems like this process became a standby. " +
+ "Ignoring the failure. The new leader will reprocess this notification event");
+ }
+ }
+ private void renamePrivileges(String oldDbName, String oldTableName, String newDbName, String newTableName) throws
+ SentryNoSuchObjectException, SentryInvalidInputException, SentryStandbyException{
+ TSentryAuthorizable oldAuthorizable = new TSentryAuthorizable(hiveInstance);
+ oldAuthorizable.setDb(oldDbName);
+ oldAuthorizable.setTable(oldTableName);
+ TSentryAuthorizable newAuthorizable = new TSentryAuthorizable(hiveInstance);
+ newAuthorizable.setDb(newDbName);
+ newAuthorizable.setTable(newTableName);
+ try {
+ sentryStore.renamePrivilege(oldAuthorizable, newAuthorizable);
+ } catch (SentryStandbyException e) {
+ LOGGER.error("Seems like this process became a standby. " +
+ "Ignoring the failure. The new leader will reprocess this notification event");
+ }
+ }
+}