You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sentry.apache.org by ha...@apache.org on 2016/06/23 19:23:25 UTC
[2/2] sentry git commit: SENTRY-1316: Implement Sentry leadership
election (Colin Patrick McCabe, Reviewed by Sravya Tirukkovalur, Hao Hao)
SENTRY-1316: Implement Sentry leadership election (Colin Patrick McCabe, Reviewed by Sravya Tirukkovalur, Hao Hao)
Change-Id: Ib7dc6470694a2f06def90039f321c28cd72952ae
Project: http://git-wip-us.apache.org/repos/asf/sentry/repo
Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/5630fc5c
Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/5630fc5c
Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/5630fc5c
Branch: refs/heads/sentry-ha-redesign
Commit: 5630fc5cecfa7720c372f84ebb0bdc059ebc2c2e
Parents: 24b686e
Author: hahao <ha...@cloudera.com>
Authored: Thu Jun 23 12:22:15 2016 -0700
Committer: hahao <ha...@cloudera.com>
Committed: Thu Jun 23 12:22:15 2016 -0700
----------------------------------------------------------------------
.../hdfs/SentryHDFSServiceClientFactory.java | 14 +-
.../hdfs/ha/HdfsHAClientInvocationHandler.java | 142 ------------
.../sentry/hdfs/MetastorePluginWithHA.java | 99 ---------
.../org/apache/sentry/hdfs/UpdateForwarder.java | 10 +-
.../sentry/hdfs/UpdateForwarderWithHA.java | 130 -----------
.../apache/sentry/hdfs/TestPluginCacheSync.java | 151 -------------
.../db/service/persistent/ServiceManager.java | 97 --------
.../thrift/HAClientInvocationHandler.java | 139 ------------
.../sentry/service/thrift/LeaderStatus.java | 134 ++++++++++++
.../service/thrift/LeaderStatusAdaptor.java | 170 ++++++++++++++
.../sentry/service/thrift/SentryService.java | 25 ++-
.../thrift/SentryServiceClientFactory.java | 8 +-
.../thrift/SentryServiceClientPoolFactory.java | 13 +-
.../sentry/service/thrift/ServiceConstants.java | 8 +-
.../persistent/TestSentryServiceDiscovery.java | 123 -----------
.../TestSentryServerForHaWithoutKerberos.java | 219 -------------------
...estSentryServerForPoolHAWithoutKerberos.java | 36 ---
.../TestSentryServiceForHAWithKerberos.java | 75 -------
.../TestSentryServiceForPoolHAWithKerberos.java | 36 ---
.../sentry/service/thrift/TestLeaderStatus.java | 219 +++++++++++++++++++
.../sentry/tests/e2e/ha/TestHaEnd2End.java | 171 ---------------
.../AbstractTestWithStaticConfiguration.java | 2 +-
22 files changed, 555 insertions(+), 1466 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sentry/blob/5630fc5c/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/SentryHDFSServiceClientFactory.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/SentryHDFSServiceClientFactory.java b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/SentryHDFSServiceClientFactory.java
index 6c9c8bb..2a18b15 100644
--- a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/SentryHDFSServiceClientFactory.java
+++ b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/SentryHDFSServiceClientFactory.java
@@ -17,11 +17,7 @@
*/
package org.apache.sentry.hdfs;
-import java.lang.reflect.Proxy;
-
import org.apache.hadoop.conf.Configuration;
-import org.apache.sentry.hdfs.ha.HdfsHAClientInvocationHandler;
-import org.apache.sentry.service.thrift.ServiceConstants.ServerConfig;
/**
* Client factory to create normal client or proxy with HA invocation handler
@@ -34,15 +30,7 @@ public class SentryHDFSServiceClientFactory {
public static SentryHDFSServiceClient create(Configuration conf)
throws Exception {
- boolean haEnabled = conf.getBoolean(ServerConfig.SENTRY_HA_ENABLED, false);
- if (haEnabled) {
- return (SentryHDFSServiceClient) Proxy.newProxyInstance(
- SentryHDFSServiceClientDefaultImpl.class.getClassLoader(),
- SentryHDFSServiceClientDefaultImpl.class.getInterfaces(),
- new HdfsHAClientInvocationHandler(conf));
- } else {
- return new SentryHDFSServiceClientDefaultImpl(conf);
- }
+ return new SentryHDFSServiceClientDefaultImpl(conf);
}
}
http://git-wip-us.apache.org/repos/asf/sentry/blob/5630fc5c/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/ha/HdfsHAClientInvocationHandler.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/ha/HdfsHAClientInvocationHandler.java b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/ha/HdfsHAClientInvocationHandler.java
deleted file mode 100644
index 6138b8c..0000000
--- a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/ha/HdfsHAClientInvocationHandler.java
+++ /dev/null
@@ -1,142 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sentry.hdfs.ha;
-
-import java.io.IOException;
-import java.lang.reflect.InvocationHandler;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-import java.net.InetSocketAddress;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.security.SecurityUtil;
-import org.apache.curator.x.discovery.ServiceInstance;
-import org.apache.sentry.hdfs.SentryHDFSServiceClientDefaultImpl;
-import org.apache.sentry.hdfs.SentryHdfsServiceException;
-import org.apache.sentry.hdfs.ServiceConstants;
-import org.apache.sentry.provider.db.service.persistent.HAContext;
-import org.apache.sentry.provider.db.service.persistent.ServiceManager;
-import org.apache.sentry.service.thrift.ServiceConstants.ServerConfig;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.base.Preconditions;
-
-public class HdfsHAClientInvocationHandler implements InvocationHandler {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(HdfsHAClientInvocationHandler.class);
-
- private final Configuration conf;
- private ServiceManager manager;
- private ServiceInstance<Void> currentServiceInstance;
- private SentryHDFSServiceClientDefaultImpl client = null;
-
- private static final String THRIFT_EXCEPTION_MESSAGE = "Thrift exception occured ";
- public static final String SENTRY_HA_ERROR_MESSAGE = "No Sentry server available. "
- + "Please ensure that at least one Sentry server is online";
-
- public HdfsHAClientInvocationHandler(Configuration conf) throws Exception {
- this.conf = conf;
- checkClientConf();
- }
-
- @Override
- public Object invoke(Object proxy, Method method, Object[] args) throws
- SentryHdfsServiceException {
- Object result = null;
- try {
- if (!method.isAccessible()) {
- method.setAccessible(true);
- }
- // The client is initialized in the first call instead of constructor.
- // This way we can propagate the connection exception to caller cleanly
- if (client == null) {
- renewSentryClient();
- }
- result = method.invoke(client, args);
- } catch (IllegalAccessException e) {
- throw new SentryHdfsServiceException(e.getMessage(), e.getCause());
- } catch (InvocationTargetException e) {
- if (!(e.getTargetException() instanceof SentryHdfsServiceException)) {
- throw new SentryHdfsServiceException("Error in Sentry HDFS client",
- e.getTargetException());
- } else {
- LOGGER.warn(THRIFT_EXCEPTION_MESSAGE + ": Error in connect current" +
- " service, will retry other service.", e);
- if (client != null) {
- client.close();
- client = null;
- }
- throw (SentryHdfsServiceException) e.getTargetException();
- }
- } catch (IOException e1) {
- // close() doesn't throw exception we supress that in case of connection
- // loss. Changing SentryPolicyServiceClient#close() to throw an
- // exception would be a backward incompatible change for Sentry clients.
- if ("close".equals(method.getName())) {
- return null;
- }
- throw new SentryHdfsServiceException(
- "Error connecting to sentry service " + e1.getMessage(), e1);
- }
- return result;
- }
-
- // Retrieve the new connection endpoint from ZK and connect to new server
- private void renewSentryClient() throws IOException {
- try {
- manager = new ServiceManager(HAContext.getHAContext(conf));
- } catch (Exception e1) {
- throw new IOException("Failed to extract Sentry node info from zookeeper", e1);
- }
-
- try {
- while (true) {
- currentServiceInstance = manager.getServiceInstance();
- if (currentServiceInstance == null) {
- throw new IOException(SENTRY_HA_ERROR_MESSAGE);
- }
- InetSocketAddress serverAddress =
- ServiceManager.convertServiceInstance(currentServiceInstance);
- conf.set(ServiceConstants.ClientConfig.SERVER_RPC_ADDRESS, serverAddress.getHostName());
- conf.setInt(ServiceConstants.ClientConfig.SERVER_RPC_PORT, serverAddress.getPort());
- try {
- client = new SentryHDFSServiceClientDefaultImpl(conf);
- LOGGER.info("Sentry Client using server " + serverAddress.getHostName() +
- ":" + serverAddress.getPort());
- break;
- } catch (IOException e) {
- manager.reportError(currentServiceInstance);
- LOGGER.info("Transport exception while opening transport:", e, e.getMessage());
- }
- }
- } finally {
- manager.close();
- }
- }
-
- private void checkClientConf() {
- if (conf.getBoolean(ServerConfig.SENTRY_HA_ZOOKEEPER_SECURITY,
- ServerConfig.SENTRY_HA_ZOOKEEPER_SECURITY_DEFAULT)) {
- String serverPrincipal = Preconditions.checkNotNull(conf.get(ServerConfig.PRINCIPAL),
- ServerConfig.PRINCIPAL + " is required");
- Preconditions.checkArgument(serverPrincipal.contains(SecurityUtil.HOSTNAME_PATTERN),
- ServerConfig.PRINCIPAL + " : " + serverPrincipal + " should contain " + SecurityUtil.HOSTNAME_PATTERN);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/sentry/blob/5630fc5c/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/MetastorePluginWithHA.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/MetastorePluginWithHA.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/MetastorePluginWithHA.java
deleted file mode 100644
index 6476a01..0000000
--- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/MetastorePluginWithHA.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sentry.hdfs;
-
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent;
-import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.sentry.hdfs.ServiceConstants.ServerConfig;
-import org.apache.sentry.provider.db.SentryPolicyStorePlugin.SentryPluginException;
-import org.apache.sentry.binding.metastore.MetastoreAuthzBindingBase;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class MetastorePluginWithHA extends MetastorePlugin {
- private static final Logger LOGGER = LoggerFactory
- .getLogger(MetastorePluginWithHA.class);
- public static class SentryMetastoreHACacheListener implements PathChildrenCacheListener {
- private MetastorePluginWithHA metastorePlugin;
-
- public SentryMetastoreHACacheListener(MetastorePluginWithHA metastorePlugin) {
- this.metastorePlugin = metastorePlugin;
- }
-
- @Override
- public void childEvent(CuratorFramework client, PathChildrenCacheEvent event)
- throws Exception {
- switch ( event.getType() ) {
- case CHILD_ADDED:
- PathsUpdate newUpdate = new PathsUpdate();
- PluginCacheSyncUtil.setUpdateFromChildEvent(event, newUpdate);
- metastorePlugin.processCacheNotification(newUpdate);
- break;
- case INITIALIZED:
- case CHILD_UPDATED:
- case CHILD_REMOVED:
- break;
- case CONNECTION_RECONNECTED:
- MetastoreAuthzBindingBase.setSentryCacheOutOfSync(false);
- break;
- case CONNECTION_SUSPENDED:
- case CONNECTION_LOST:
- MetastoreAuthzBindingBase.setSentryCacheOutOfSync(true);
- break;
- default:
- break;
- }
- }
- }
-
- private String zkPath;
- private PluginCacheSyncUtil pluginCacheSync;
-
- public MetastorePluginWithHA(Configuration conf, Configuration sentryConfig) throws Exception {
- super(conf, sentryConfig);
- zkPath = sentryConfig.get(ServerConfig.SENTRY_METASTORE_HA_ZOOKEEPER_NAMESPACE,
- ServerConfig.SENTRY_METASTORE_HA_ZOOKEEPER_NAMESPACE_DEFAULT);
-
- pluginCacheSync = new PluginCacheSyncUtil(zkPath, sentryConfig,
- new SentryMetastoreHACacheListener(this));
- // start seq# from the last global seq
- seqNum.set(pluginCacheSync.getUpdateCounter());
- MetastorePlugin.lastSentSeqNum = seqNum.get();
- }
-
- @Override
- protected void processUpdate(PathsUpdate update) {
- try {
- // push to ZK in order to keep the metastore local cache in sync
- pluginCacheSync.handleCacheUpdate(update);
-
- // notify Sentry. Note that Sentry service already has a cache
- // sync mechanism to replicate this update to all other Sentry servers
- notifySentry(update);
- } catch (SentryPluginException e) {
- LOGGER.error("Error pushing update to cache", e);
- }
- }
-
- // apply the update to local cache
- private void processCacheNotification(PathsUpdate update) {
- super.applyLocal(update);
- }
-}
http://git-wip-us.apache.org/repos/asf/sentry/blob/5630fc5c/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateForwarder.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateForwarder.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateForwarder.java
index 7387281..ea1c8f6 100644
--- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateForwarder.java
+++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateForwarder.java
@@ -31,7 +31,6 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.hadoop.conf.Configuration;
import org.apache.sentry.provider.db.SentryPolicyStorePlugin.SentryPluginException;
-import org.apache.sentry.provider.db.service.persistent.HAContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -98,13 +97,8 @@ public class UpdateForwarder<K extends Updateable.Update> implements
public static <K extends Updateable.Update> UpdateForwarder<K> create(Configuration conf,
Updateable<K> updateable, K update, ExternalImageRetriever<K> imageRetreiver,
int maxUpdateLogSize, int initUpdateRetryDelay) throws SentryPluginException {
- if (HAContext.isHaEnabled(conf)) {
- return new UpdateForwarderWithHA<K>(conf, updateable, update, imageRetreiver,
- maxUpdateLogSize, initUpdateRetryDelay);
- } else {
- return new UpdateForwarder<K>(conf, updateable, imageRetreiver,
- maxUpdateLogSize, initUpdateRetryDelay);
- }
+ return new UpdateForwarder<K>(conf, updateable, imageRetreiver,
+ maxUpdateLogSize, initUpdateRetryDelay);
}
private void spawnInitialUpdater(final Updateable<K> updateable,
http://git-wip-us.apache.org/repos/asf/sentry/blob/5630fc5c/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateForwarderWithHA.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateForwarderWithHA.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateForwarderWithHA.java
deleted file mode 100644
index 574627c..0000000
--- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateForwarderWithHA.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sentry.hdfs;
-
-import java.io.IOException;
-import java.util.LinkedList;
-
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent;
-import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.sentry.hdfs.ServiceConstants.ServerConfig;
-import org.apache.sentry.provider.db.SentryPolicyStorePlugin.SentryPluginException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class UpdateForwarderWithHA<K extends Updateable.Update> extends
-UpdateForwarder<K> implements Updateable<K> {
- private static final Logger LOGGER = LoggerFactory.getLogger(UpdateForwarderWithHA.class);
- private static final String UPDATABLE_TYPE_NAME = "ha_update_forwarder";
-
- public static class SentryHAPathChildrenCacheListener<K extends Updateable.Update>
- implements PathChildrenCacheListener {
- private final LinkedList<K> updateLog;
- private final K baseUpdate;
- private final UpdateForwarderWithHA<K> updateForwarder;
-
- public SentryHAPathChildrenCacheListener(LinkedList<K> updateLog,
- K baseUpdate, UpdateForwarderWithHA<K> updateForwarder) {
- this.updateLog = updateLog;
- this.baseUpdate = baseUpdate;
- this.updateForwarder = updateForwarder;
- }
-
- @Override
- public synchronized void childEvent(CuratorFramework client,
- PathChildrenCacheEvent event) throws Exception {
- switch ( event.getType() ) {
- case CHILD_ADDED:
- K newUpdate = (K) baseUpdate.getClass().newInstance();
- PluginCacheSyncUtil.setUpdateFromChildEvent(event, newUpdate);
- updateForwarder.postNotificationToLog(newUpdate);
- break;
- case INITIALIZED:
- case CHILD_UPDATED:
- case CHILD_REMOVED:
- break;
- case CONNECTION_RECONNECTED:
- // resume the node
- SentryPlugin.instance.setOutOfSync(false);
- break;
- case CONNECTION_SUSPENDED:
- case CONNECTION_LOST:
- // suspend the node
- SentryPlugin.instance.setOutOfSync(true);
- break;
- default:
- break;
- }
- }
- }
-
- private final String zkPath;
- private final PluginCacheSyncUtil pluginCacheSync;
-
- public UpdateForwarderWithHA(Configuration conf, Updateable<K> updateable, K baseUpdate,
- ExternalImageRetriever<K> imageRetreiver, int updateLogSize) throws SentryPluginException {
- this(conf, updateable, baseUpdate, imageRetreiver, updateLogSize, INIT_UPDATE_RETRY_DELAY);
- }
-
- public UpdateForwarderWithHA(Configuration conf, Updateable<K> updateable, K baseUpdate,
- ExternalImageRetriever<K> imageRetreiver, int updateLogSize,
- int initUpdateRetryDelay) throws SentryPluginException {
- super(conf, updateable, imageRetreiver, updateLogSize, initUpdateRetryDelay);
- zkPath = conf.get(ServerConfig.SENTRY_HDFS_HA_ZOOKEEPER_NAMESPACE,
- ServerConfig.SENTRY_HDFS_HA_ZOOKEEPER_NAMESPACE_DEFAULT) + "/" +
- updateable.getUpdateableTypeName();
- pluginCacheSync = new PluginCacheSyncUtil(zkPath, conf,
- new SentryHAPathChildrenCacheListener<K>(getUpdateLog(), baseUpdate,
- this));
- }
-
- @Override
- public String getUpdateableTypeName() {
- return UPDATABLE_TYPE_NAME;
- }
-
- @Override
- public void handleUpdateNotification(final K update) throws SentryPluginException {
- pluginCacheSync.handleCacheUpdate(update);
- }
-
- private void postNotificationToLog(K update) throws SentryPluginException {
- super.handleUpdateNotification(update);
- }
-
- @Override
- public void close() throws IOException {
- pluginCacheSync.close();
- }
-
- @Override
- public boolean areAllUpdatesCommited() {
- try {
- if (lastCommittedSeqNum.get() == INIT_SEQ_NUM) {
- return false;
- }
- return lastCommittedSeqNum.get() == pluginCacheSync.getUpdateCounter();
- } catch (Exception e) {
- LOGGER.error("Error loading the update counter for ZK", e);
- return true;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/sentry/blob/5630fc5c/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestPluginCacheSync.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestPluginCacheSync.java b/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestPluginCacheSync.java
deleted file mode 100644
index 132a4a3..0000000
--- a/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestPluginCacheSync.java
+++ /dev/null
@@ -1,151 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sentry.hdfs;
-
-import static org.junit.Assert.*;
-
-import java.util.List;
-
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent;
-import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener;
-import org.apache.curator.test.TestingServer;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.sentry.hdfs.TestUpdateForwarder.DummyUpdate;
-import org.apache.sentry.provider.db.service.persistent.HAContext;
-import org.apache.sentry.service.thrift.ServiceConstants.ServerConfig;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-public class TestPluginCacheSync {
- // Test PathChildrenCacheListener to track the Update event received from ZK
- public static class TestCacheListener implements PathChildrenCacheListener {
- private DummyUpdate dummyUpdate;
- private boolean recievedEvent = false;
-
- @Override
- public void childEvent(CuratorFramework client, PathChildrenCacheEvent event)
- throws Exception {
- switch (event.getType()) {
- case CHILD_ADDED:
- DummyUpdate newUpdate = new DummyUpdate();
- PluginCacheSyncUtil.setUpdateFromChildEvent(event, newUpdate);
- dummyUpdate = newUpdate;
- recievedEvent = true;
- break;
- default:
- break;
- }
- }
-
- public DummyUpdate getDummyUpdate() {
- return dummyUpdate;
- }
-
- public boolean isRecievedEvent() {
- return recievedEvent;
- }
-
- public void setRecievedEvent(boolean recievedEvent) {
- this.recievedEvent = recievedEvent;
- }
- }
-
- private static final String TEST_ZPATH = "/test";
- private static TestingServer testServer;
- private static Configuration conf;
-
- private PluginCacheSyncUtil pluginCache;
-
- @BeforeClass
- public static void preSetup() throws Exception {
- testServer = new TestingServer();
- testServer.start();
- conf = new Configuration();
- conf.set(ServerConfig.SENTRY_HA_ZOOKEEPER_QUORUM,
- testServer.getConnectString());
- }
-
- @After
- public void destroy() throws Exception {
- pluginCache.close();
- }
-
- @AfterClass
- public static void shutDown() throws Exception {
- testServer.stop();
- }
-
- /**
- * Post a dummy update to PluginCacheSync. Verify that the update is received
- * by cache via ZK sync
- * @throws Exception
- */
- @Test
- public void testCachePost() throws Exception {
- TestCacheListener cacheListener = new TestCacheListener();
- pluginCache = new PluginCacheSyncUtil(TEST_ZPATH, conf,
- cacheListener);
-
- // post an update
- DummyUpdate dummyUpdate = new DummyUpdate();
- dummyUpdate.setState("foo");
- pluginCache.handleCacheUpdate(dummyUpdate);
-
- // wait for update to sync up
- int timeLeft = 5000;
- while (!cacheListener.isRecievedEvent() && (timeLeft > 0)) {
- Thread.sleep(200);
- timeLeft -= 200;
- }
- cacheListener.setRecievedEvent(false);
- DummyUpdate newUpdate = cacheListener.getDummyUpdate();
- assertEquals(dummyUpdate.getState(), newUpdate.getState());
- }
-
- @Test
- public void pluginCacheGC() throws Exception {
- pluginCache = new PluginCacheSyncUtil(TEST_ZPATH, conf,
- new TestCacheListener());
-
- // post updates
- for (int updCount = 1; updCount <= PluginCacheSyncUtil.CACHE_GC_SIZE_THRESHOLD_HWM + 2; updCount++) {
- DummyUpdate dummyUpdate = new DummyUpdate();
- dummyUpdate.setSeqNum(updCount);
- dummyUpdate.setState("foo");
- pluginCache.handleCacheUpdate(dummyUpdate);
- }
-
- // force gc
- pluginCache.gcPluginCache(conf);
-
- // count remaining znodes
- HAContext haContext = HAContext.getHAContext(conf);
- List<String> znodeList = haContext.getCuratorFramework().getChildren()
- .forPath(TEST_ZPATH + "/cache");
- assertFalse(znodeList.isEmpty());
- assertFalse(znodeList.contains(String
- .valueOf(PluginCacheSyncUtil.GC_COUNTER_INIT_VALUE)));
- assertFalse(znodeList.contains(String
- .valueOf(PluginCacheSyncUtil.GC_COUNTER_INIT_VALUE + 1)));
- assertTrue(znodeList.contains(String
- .valueOf(PluginCacheSyncUtil.CACHE_GC_SIZE_THRESHOLD_HWM)));
- }
-}
http://git-wip-us.apache.org/repos/asf/sentry/blob/5630fc5c/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/ServiceManager.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/ServiceManager.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/ServiceManager.java
deleted file mode 100644
index 9f921d4..0000000
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/ServiceManager.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.sentry.provider.db.service.persistent;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-
-import org.apache.curator.x.discovery.ServiceDiscovery;
-import org.apache.curator.x.discovery.ServiceDiscoveryBuilder;
-import org.apache.curator.x.discovery.ServiceInstance;
-import org.apache.curator.x.discovery.ServiceProvider;
-import org.apache.curator.x.discovery.details.InstanceSerializer;
-import org.apache.hadoop.net.NetUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/***
- * ServerManager handles registration of the Sentry service for Curator service
- * discovery. Each server registers with ZK and add its host:port details which
- * is used by the clients to discover available servers
- */
-public class ServiceManager {
- private static final Logger LOGGER = LoggerFactory
- .getLogger(ServiceManager.class);
- private HAContext haContext;
- private ServiceProvider<Void> serviceProvider;
- private ServiceDiscovery<Void> serviceDiscovery;
-
- public ServiceManager(HAContext haContext) throws IOException {
- this.haContext = haContext;
- init();
- }
-
- private void init() throws IOException {
- try {
- haContext.startCuratorFramework();
- InstanceSerializer<Void> instanceSerializer = new FixedJsonInstanceSerializer<Void>(Void.class);
- serviceDiscovery = ServiceDiscoveryBuilder.<Void>builder(Void.class)
- .basePath(HAContext.SENTRY_SERVICE_REGISTER_NAMESPACE)
- .serializer(instanceSerializer)
- .client(haContext.getCuratorFramework())
- .build();
- serviceDiscovery.start();
- serviceProvider = serviceDiscovery
- .serviceProviderBuilder()
- .serviceName(HAContext.SENTRY_SERVICE_REGISTER_NAMESPACE)
- .build();
- serviceProvider.start();
- } catch (Exception e) {
- throw new IOException(e);
- }
- }
-
- public ServiceInstance<Void> getServiceInstance() throws IOException {
- ServiceInstance<Void> service;
- try {
- service = serviceProvider.getInstance();
- return service;
- } catch (Exception e) {
- throw new IOException(e);
- }
- }
-
- public void reportError(ServiceInstance<Void> instance) {
- serviceProvider.noteError(instance);
- }
-
- public static InetSocketAddress convertServiceInstance(ServiceInstance<?> service) {
- return NetUtils.createSocketAddr(service.getAddress(),service.getPort());
- }
-
- public void close() {
- try {
- serviceProvider.close();
- serviceDiscovery.close();
- LOGGER.debug("Closed ZK resources");
- } catch (IOException e) {
- LOGGER.warn("Error closing the service manager", e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/sentry/blob/5630fc5c/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HAClientInvocationHandler.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HAClientInvocationHandler.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HAClientInvocationHandler.java
deleted file mode 100644
index d97a07e..0000000
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HAClientInvocationHandler.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sentry.service.thrift;
-
-import java.io.IOException;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-import java.net.InetSocketAddress;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.security.SecurityUtil;
-import org.apache.curator.x.discovery.ServiceInstance;
-import org.apache.sentry.core.common.exception.SentryUserException;
-import org.apache.sentry.provider.db.service.persistent.HAContext;
-import org.apache.sentry.provider.db.service.persistent.ServiceManager;
-import org.apache.sentry.provider.db.service.thrift.SentryPolicyServiceClient;
-import org.apache.sentry.provider.db.service.thrift.SentryPolicyServiceClientDefaultImpl;
-import org.apache.sentry.service.thrift.ServiceConstants.ServerConfig;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.base.Preconditions;
-
-public class HAClientInvocationHandler extends SentryClientInvocationHandler {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(HAClientInvocationHandler.class);
-
- private final Configuration conf;
- private ServiceManager manager;
- private ServiceInstance<Void> currentServiceInstance;
- private SentryPolicyServiceClient client = null;
-
- private static final String THRIFT_EXCEPTION_MESSAGE = "Thrift exception occured ";
- public static final String SENTRY_HA_ERROR_MESSAGE = "No Sentry server available. Please ensure that at least one Sentry server is online";
-
- public HAClientInvocationHandler(Configuration conf) throws Exception {
- this.conf = conf;
- checkClientConf();
- }
-
- @Override
- public Object invokeImpl(Object proxy, Method method, Object[] args) throws
- SentryUserException {
- Object result = null;
- try {
- if (!method.isAccessible()) {
- method.setAccessible(true);
- }
- // The client is initialized in the first call instead of constructor.
- // This way we can propagate the connection exception to caller cleanly
- if (client == null) {
- renewSentryClient();
- }
- result = method.invoke(client, args);
- } catch (IllegalAccessException e) {
- throw new SentryUserException(e.getMessage(), e.getCause());
- } catch (InvocationTargetException e) {
- if (e.getTargetException() instanceof SentryUserException) {
- throw (SentryUserException)e.getTargetException();
- } else {
- LOGGER.warn(THRIFT_EXCEPTION_MESSAGE + ": Error in connect current" +
- " service, will retry other service.", e);
- if (client != null) {
- client.close();
- client = null;
- }
- }
- } catch (IOException e1) {
- throw new SentryUserException("Error connecting to sentry service "
- + e1.getMessage(), e1);
- }
- return result;
- }
-
- // Retrieve the new connection endpoint from ZK and connect to new server
- private void renewSentryClient() throws IOException {
- try {
- manager = new ServiceManager(HAContext.getHAContext(conf));
- } catch (Exception e1) {
- throw new IOException("Failed to extract Sentry node info from zookeeper", e1);
- }
-
- try {
- while (true) {
- currentServiceInstance = manager.getServiceInstance();
- if (currentServiceInstance == null) {
- throw new IOException(SENTRY_HA_ERROR_MESSAGE);
- }
- InetSocketAddress serverAddress =
- ServiceManager.convertServiceInstance(currentServiceInstance);
- conf.set(ServiceConstants.ClientConfig.SERVER_RPC_ADDRESS, serverAddress.getHostName());
- conf.setInt(ServiceConstants.ClientConfig.SERVER_RPC_PORT, serverAddress.getPort());
- try {
- client = new SentryPolicyServiceClientDefaultImpl(conf);
- LOGGER.info("Sentry Client using server " + serverAddress.getHostName() +
- ":" + serverAddress.getPort());
- break;
- } catch (IOException e) {
- manager.reportError(currentServiceInstance);
- LOGGER.info("Transport exception while opening transport:", e, e.getMessage());
- }
- }
- } finally {
- manager.close();
- }
- }
-
- private void checkClientConf() {
- if (conf.getBoolean(ServerConfig.SENTRY_HA_ZOOKEEPER_SECURITY,
- ServerConfig.SENTRY_HA_ZOOKEEPER_SECURITY_DEFAULT)) {
- String serverPrincipal = Preconditions.checkNotNull(conf.get(ServerConfig.PRINCIPAL),
- ServerConfig.PRINCIPAL + " is required");
- Preconditions.checkArgument(serverPrincipal.contains(SecurityUtil.HOSTNAME_PATTERN),
- ServerConfig.PRINCIPAL + " : " + serverPrincipal + " should contain " + SecurityUtil.HOSTNAME_PATTERN);
- }
- }
-
- @Override
- public void close() {
- if (client != null) {
- client.close();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/sentry/blob/5630fc5c/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/LeaderStatus.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/LeaderStatus.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/LeaderStatus.java
new file mode 100644
index 0000000..e846766
--- /dev/null
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/LeaderStatus.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sentry.service.thrift;
+
+import org.apache.commons.codec.binary.Hex;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.security.SecureRandom;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static org.apache.sentry.service.thrift.ServiceConstants.ServerConfig.SENTRY_HA_ENABLED;
+import static org.apache.sentry.service.thrift.ServiceConstants.ServerConfig.SENTRY_HA_ENABLED_DEFAULT;
+
+/**
+ * Determines the leadership status of the Sentry daemon.
+ * It handles both highly-available and non-highly-available configurations.
+ */
+final class LeaderStatus implements Closeable {
+ private static final Log LOG = LogFactory.getLog(LeaderStatus.class);
+
+ /**
+ * Callback functions which are invoked when the leader status changes.
+ */
+ interface Listener {
+ /**
+ * Attempt to become the leader.
+ *
+ * @throws Exception On error. If an exception is thrown, we will
+ * relinquish leadership.
+ */
+ void becomeActive() throws Exception;
+
+ /**
+ * Become the standby. All exceptions thrown from this function will
+ * be ignored.
+ */
+ void becomeStandby();
+ }
+
+ /**
+ * The listener to invoke when our leadership status changes.
+ */
+ private final Listener listener;
+
+ /**
+ * The unique ID of this potential leader.
+ */
+ private final String incarnationId;
+
+ /**
+ * The LeaderStatusThread, or null if HA is disabled.
+ */
+ private final LeaderStatusAdaptor leaderStatusAdaptor;
+
+ /**
+ * True if this object has been closed.<p/>
+ *
+ * This is an AtomicBoolean so that multiple calls to close only result in one
+ * close action.
+ */
+ private final AtomicBoolean closed = new AtomicBoolean(false);
+
+ /**
+ * Generate a 128-bit random ID.
+ */
+ static String generateIncarnationId() {
+ SecureRandom srand = new SecureRandom();
+ byte[] buf = new byte[32];
+ srand.nextBytes(buf);
+ return "sentry_" + Hex.encodeHexString(buf);
+ }
+
+ LeaderStatus(Listener listener, Configuration conf) throws Exception {
+ this.listener = listener;
+ this.incarnationId = generateIncarnationId();
+ boolean isHa = conf.
+ getBoolean(SENTRY_HA_ENABLED, SENTRY_HA_ENABLED_DEFAULT);
+ if (isHa) {
+ this.leaderStatusAdaptor = new LeaderStatusAdaptor(incarnationId, conf, listener);
+ } else {
+ LOG.info("LeaderStatus(incarnationId=" + incarnationId +
+ "): HA is disabled.");
+ this.leaderStatusAdaptor = null;
+ }
+ }
+
+ public String getIncarnationId() {
+ return incarnationId;
+ }
+
+ public void start() throws Exception {
+ if (this.leaderStatusAdaptor != null) {
+ this.leaderStatusAdaptor.start();
+ } else {
+ this.listener.becomeActive();
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (closed.compareAndSet(false, true)) {
+ if (leaderStatusAdaptor != null) {
+ // Shut down in the HA case.
+ leaderStatusAdaptor.close();
+ } else {
+ // Shut down in the non-HA case.
+ try {
+ listener.becomeStandby();
+ } catch (Throwable t) {
+ LOG.error("becomeStandby: " + incarnationId +
+ " threw an unexpected exception", t);
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/sentry/blob/5630fc5c/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/LeaderStatusAdaptor.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/LeaderStatusAdaptor.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/LeaderStatusAdaptor.java
new file mode 100644
index 0000000..80a6571
--- /dev/null
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/LeaderStatusAdaptor.java
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sentry.service.thrift;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.framework.recipes.leader.LeaderSelector;
+import org.apache.curator.framework.recipes.leader.LeaderSelectorListenerAdapter;
+import org.apache.curator.retry.ExponentialBackoffRetry;
+import org.apache.hadoop.conf.Configuration;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.ReentrantLock;
+
+import static org.apache.sentry.service.thrift.ServiceConstants.ServerConfig.SENTRY_HA_ZOOKEEPER_QUORUM;
+import static org.apache.sentry.service.thrift.ServiceConstants.ServerConfig.SENTRY_HA_ZOOKEEPER_QUORUM_DEFAULT;
+import static org.apache.sentry.service.thrift.ServiceConstants.ServerConfig.SENTRY_HA_ZOOKEEPER_NAMESPACE;
+import static org.apache.sentry.service.thrift.ServiceConstants.ServerConfig.SENTRY_HA_ZOOKEEPER_NAMESPACE_DEFAULT;
+
+/**
+ * Determines the leadership status of the Sentry daemon.
+ */
+final class LeaderStatusAdaptor
+ extends LeaderSelectorListenerAdapter implements Closeable {
+ private static final Log LOG =
+ LogFactory.getLog(LeaderStatusAdaptor.class);
+
+ private final String LEADER_SELECTOR_SUFFIX = "leader";
+
+ /**
+ * The ZooKeeper path prefix to use.
+ */
+ private final String zkNamespace;
+
+ /**
+ * The Curator framework object.
+ */
+ private final CuratorFramework framework;
+
+ /**
+ * The listener which we should notify about HA state changes.
+ */
+ private final LeaderStatus.Listener listener;
+
+ /**
+ * The Curator LeaderSelector object.
+ */
+ private final LeaderSelector leaderSelector;
+
+ /**
+ * The lock which protects isActive.
+ */
+ private final ReentrantLock lock = new ReentrantLock();
+
+ /**
+ * A condition variable which the takeLeadership function will wait on.
+ */
+ private final Condition cond = lock.newCondition();
+
+ /**
+ * The number of times this incarnation has become the leader.
+ */
+ private long becomeLeaderCount = 0;
+
+ /**
+ * True only if this incarnation is currently active.
+ */
+ private boolean isActive = false;
+
+ LeaderStatusAdaptor(String incarnationId, Configuration conf,
+ LeaderStatus.Listener listener) {
+ this.zkNamespace = conf.get(SENTRY_HA_ZOOKEEPER_NAMESPACE,
+ SENTRY_HA_ZOOKEEPER_NAMESPACE_DEFAULT);
+ String zkServers = conf.get(SENTRY_HA_ZOOKEEPER_QUORUM,
+ SENTRY_HA_ZOOKEEPER_QUORUM_DEFAULT);
+ if ((zkServers == null) || (zkServers.trim().isEmpty())) {
+ throw new RuntimeException("You must configure some ZooKeeper " +
+ "servers via " + SENTRY_HA_ZOOKEEPER_QUORUM + " when enabling HA");
+ }
+ this.framework = CuratorFrameworkFactory.newClient(zkServers,
+ new ExponentialBackoffRetry(1000, 3));
+ this.framework.start();
+ this.listener = listener;
+ this.leaderSelector = new LeaderSelector(this.framework,
+ this.zkNamespace + "/" + LEADER_SELECTOR_SUFFIX, this);
+ this.leaderSelector.setId(incarnationId);
+ this.leaderSelector.autoRequeue();
+ LOG.info("Created LeaderStatusAdaptor(zkNamespace=" + zkNamespace +
+ ", incarnationId=" + incarnationId +
+ ", zkServers='" + zkServers + "')");
+ }
+
+ public void start() {
+ this.leaderSelector.start();
+ }
+
+ @Override
+ public void close() throws IOException {
+ leaderSelector.close();
+ }
+
+ /**
+ * @return true if this client is the current leader.
+ */
+ public boolean isActive() {
+ lock.lock();
+ try {
+ return isActive;
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ /**
+ * Deactivate the current client, if it is active.
+ */
+ public void deactivate() {
+ lock.lock();
+ try {
+ if (isActive) {
+ isActive = false;
+ cond.signal();
+ }
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ @Override
+ public void takeLeadership(CuratorFramework client) throws Exception {
+ lock.lock();
+ try {
+ isActive = true;
+ becomeLeaderCount++;
+ LOG.info("SentryLeaderSelectorClient: becoming active. " +
+ "becomeLeaderCount=" + becomeLeaderCount);
+ listener.becomeActive();
+ while (isActive) {
+ cond.await();
+ }
+ } finally {
+ isActive = false;
+ LOG.info("SentryLeaderSelectorClient: becoming standby");
+ try {
+ listener.becomeStandby();
+ } catch (Throwable t) {
+ LOG.error("becomeStandby threw unexpected exception", t);
+ }
+ lock.unlock();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/sentry/blob/5630fc5c/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java
index 6883bf4..809af06 100644
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java
@@ -95,8 +95,9 @@ public class SentryService implements Callable {
private SentryWebServer sentryWebServer;
private long maxMessageSize;
private final boolean isHA;
- private boolean isActive;
+ private volatile boolean isActive = false;
SentryMetrics sentryMetrics;
+ private final LeaderStatus leaderStatus;
public SentryService(Configuration conf) {
this.conf = conf;
@@ -143,8 +144,6 @@ public class SentryService implements Callable {
}
isHA = conf.getBoolean(ServerConfig.SENTRY_HA_ENABLED,
ServerConfig.SENTRY_HA_ENABLED_DEFAULT);
- //setting isActive to true for now, until we have Sentry HA implemented
- isActive = true;
serviceExecutor = Executors.newSingleThreadExecutor(new ThreadFactory() {
private int count = 0;
@@ -154,6 +153,25 @@ public class SentryService implements Callable {
+ (count++));
}
});
+ try {
+ leaderStatus = new LeaderStatus(
+ new LeaderStatus.Listener() {
+ @Override
+ public void becomeActive() throws Exception {
+ LOGGER.info("Activating " + leaderStatus.getIncarnationId());
+ isActive = true;
+ }
+
+ @Override
+ public void becomeStandby() {
+ LOGGER.info("Deactivating " + leaderStatus.getIncarnationId());
+ isActive = false;
+ }
+ }, conf);
+ leaderStatus.start(); // TODO: move this into call?
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
webServerPort = conf.getInt(ServerConfig.SENTRY_WEB_PORT, ServerConfig.SENTRY_WEB_PORT_DEFAULT);
status = Status.NOT_STARTED;
}
@@ -289,6 +307,7 @@ public class SentryService implements Callable {
public synchronized void stop() throws Exception{
MultiException exception = null;
LOGGER.info("Attempting to stop...");
+ leaderStatus.close();
if (isRunning()) {
LOGGER.info("Attempting to stop sentry thrift service...");
try {
http://git-wip-us.apache.org/repos/asf/sentry/blob/5630fc5c/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceClientFactory.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceClientFactory.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceClientFactory.java
index 48ee66a..56d774b 100644
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceClientFactory.java
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceClientFactory.java
@@ -32,21 +32,15 @@ public final class SentryServiceClientFactory {
}
public static SentryPolicyServiceClient create(Configuration conf) throws Exception {
- boolean haEnabled = conf.getBoolean(ClientConfig.SERVER_HA_ENABLED, false);
boolean pooled = conf.getBoolean(ClientConfig.SENTRY_POOL_ENABLED, false);
if (pooled) {
return (SentryPolicyServiceClient) Proxy
.newProxyInstance(SentryPolicyServiceClientDefaultImpl.class.getClassLoader(),
SentryPolicyServiceClientDefaultImpl.class.getInterfaces(),
new PoolClientInvocationHandler(conf));
- } else if (haEnabled) {
- return (SentryPolicyServiceClient) Proxy
- .newProxyInstance(SentryPolicyServiceClientDefaultImpl.class.getClassLoader(),
- SentryPolicyServiceClientDefaultImpl.class.getInterfaces(),
- new HAClientInvocationHandler(conf));
} else {
return new SentryPolicyServiceClientDefaultImpl(conf);
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/sentry/blob/5630fc5c/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceClientPoolFactory.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceClientPoolFactory.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceClientPoolFactory.java
index 3a38b24..afea78a 100644
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceClientPoolFactory.java
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceClientPoolFactory.java
@@ -18,15 +18,12 @@
package org.apache.sentry.service.thrift;
-import java.lang.reflect.Proxy;
-
import org.apache.commons.pool2.BasePooledObjectFactory;
import org.apache.commons.pool2.PooledObject;
import org.apache.commons.pool2.impl.DefaultPooledObject;
import org.apache.hadoop.conf.Configuration;
import org.apache.sentry.provider.db.service.thrift.SentryPolicyServiceClient;
import org.apache.sentry.provider.db.service.thrift.SentryPolicyServiceClientDefaultImpl;
-import org.apache.sentry.service.thrift.ServiceConstants.ClientConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -48,15 +45,7 @@ public class SentryServiceClientPoolFactory extends BasePooledObjectFactory<Sent
@Override
public SentryPolicyServiceClient create() throws Exception {
LOGGER.debug("Creating Sentry Service Client...");
- boolean haEnabled = conf.getBoolean(ClientConfig.SERVER_HA_ENABLED, false);
- if (haEnabled) {
- return (SentryPolicyServiceClient) Proxy
- .newProxyInstance(SentryPolicyServiceClientDefaultImpl.class.getClassLoader(),
- SentryPolicyServiceClientDefaultImpl.class.getInterfaces(),
- new HAClientInvocationHandler(conf));
- } else {
- return new SentryPolicyServiceClientDefaultImpl(conf);
- }
+ return new SentryPolicyServiceClientDefaultImpl(conf);
}
@Override
http://git-wip-us.apache.org/repos/asf/sentry/blob/5630fc5c/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/ServiceConstants.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/ServiceConstants.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/ServiceConstants.java
index 32a4044..0ab8192 100644
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/ServiceConstants.java
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/ServiceConstants.java
@@ -115,13 +115,13 @@ public class ServiceConstants {
public static final String SENTRY_HA_ZOOKEEPER_SECURITY = SENTRY_HA_ZK_PROPERTY_PREFIX + "security";
public static final boolean SENTRY_HA_ZOOKEEPER_SECURITY_DEFAULT = false;
public static final String SENTRY_HA_ZOOKEEPER_QUORUM = SENTRY_HA_ZK_PROPERTY_PREFIX + "quorum";
- public static final String SENTRY_HA_ZOOKEEPER_QUORUM_DEFAULT = "localhost:2181";
+ public static final String SENTRY_HA_ZOOKEEPER_QUORUM_DEFAULT = "";
public static final String SENTRY_HA_ZOOKEEPER_RETRIES_MAX_COUNT = SENTRY_HA_ZK_PROPERTY_PREFIX + "session.retries.max.count";
public static final int SENTRY_HA_ZOOKEEPER_RETRIES_MAX_COUNT_DEFAULT = 3;
public static final String SENTRY_HA_ZOOKEEPER_SLEEP_BETWEEN_RETRIES_MS = SENTRY_HA_ZK_PROPERTY_PREFIX + "session.sleep.between.retries.ms";
public static final int SENTRY_HA_ZOOKEEPER_SLEEP_BETWEEN_RETRIES_MS_DEFAULT = 100;
public static final String SENTRY_HA_ZOOKEEPER_NAMESPACE = SENTRY_HA_ZK_PROPERTY_PREFIX + "namespace";
- public static final String SENTRY_HA_ZOOKEEPER_NAMESPACE_DEFAULT = "sentry";
+ public static final String SENTRY_HA_ZOOKEEPER_NAMESPACE_DEFAULT = "/sentry";
// principal and keytab for client to be able to connect to secure ZK. Needed for Sentry HA with secure ZK
public static final String SERVER_HA_ZOOKEEPER_CLIENT_PRINCIPAL = "sentry.zookeeper.client.principal";
public static final String SERVER_HA_ZOOKEEPER_CLIENT_KEYTAB = "sentry.zookeeper.client.keytab";
@@ -200,8 +200,8 @@ public class ServiceConstants {
public static final int SERVER_RPC_CONN_TIMEOUT_DEFAULT = 200000;
// HA configuration
- public static final String SERVER_HA_ENABLED = "sentry.ha.enabled";
- public static final boolean SERVER_HA_ENABLED_DEFAULT = ServerConfig.SENTRY_HA_ENABLED_DEFAULT;
+ public static final String SENTRY_HA_ENABLED = "sentry.ha.enabled";
+ public static final boolean SENTRY_HA_ENABLED_DEFAULT = ServerConfig.SENTRY_HA_ENABLED_DEFAULT;
public static final String SENTRY_HA_ZOOKEEPER_QUORUM = ServerConfig.SENTRY_HA_ZOOKEEPER_QUORUM;
public static final String SERVER_HA_ZOOKEEPER_QUORUM_DEFAULT = ServerConfig.SENTRY_HA_ZOOKEEPER_QUORUM_DEFAULT;
public static final String SENTRY_HA_ZOOKEEPER_NAMESPACE = ServerConfig.SENTRY_HA_ZOOKEEPER_NAMESPACE;
http://git-wip-us.apache.org/repos/asf/sentry/blob/5630fc5c/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryServiceDiscovery.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryServiceDiscovery.java b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryServiceDiscovery.java
deleted file mode 100644
index 7cbcc11..0000000
--- a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryServiceDiscovery.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.sentry.provider.db.service.persistent;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.hadoop.conf.Configuration;
-
-import org.apache.curator.test.TestingServer;
-import org.apache.curator.x.discovery.ServiceInstance;
-import org.apache.sentry.service.thrift.ServiceConstants.ServerConfig;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-import static org.junit.Assert.assertEquals;
-
-public class TestSentryServiceDiscovery {
-
- private HAContext haContext;
- private TestingServer server;
-
- @Before
- public void setup() throws Exception {
- server = new TestingServer();
- // HA conf
- Configuration conf = new Configuration(false);
- conf.set(ServerConfig.SENTRY_HA_ENABLED, "true");
- conf.set(ServerConfig.SENTRY_HA_ZOOKEEPER_NAMESPACE, "sentry-test");
- conf.set(ServerConfig.SENTRY_HA_ZOOKEEPER_QUORUM, server.getConnectString());
- haContext = HAContext.getHAContext(conf);
- }
-
- @After
- public void teardown() {
- HAContext.clearServerContext();
- if (server != null) {
- try {
- server.stop();
- } catch (IOException e) {
- }
- }
- }
-
- @Test
- public void testRegisterOneService() throws Exception {
- final String hostname = "localhost1";
- final Integer port = 123;
- ServiceRegister register = new ServiceRegister(haContext);
- register.regService(hostname, port);
- ServiceManager manager = new ServiceManager(haContext);
- ServiceInstance<Void> instance = manager.getServiceInstance();
- assertEquals("'hostname' doesn't match.", hostname, instance.getAddress());
- assertEquals("'port' doesn't match.", port, instance.getPort());
- }
-
- @Test
- public void testRegisterMultiService() throws Exception {
-
- final String hostname1 = "localhost1";
- final Integer port1 = 123;
- final String hostname2 = "localhost2";
- final Integer port2 = 456;
- final String hostname3 = "localhost3";
- final Integer port3 = 789;
-
- Map<String, Integer> servicesMap = new HashMap<String, Integer>();
- servicesMap.put(hostname1, port1);
- servicesMap.put(hostname2, port2);
- servicesMap.put(hostname3, port3);
-
- ServiceRegister register1 = new ServiceRegister(haContext);
- register1.regService(hostname1, port1);
- ServiceRegister register2 = new ServiceRegister(haContext);
- register2.regService(hostname2, port2);
- ServiceRegister register3 = new ServiceRegister(haContext);
- register3.regService(hostname3, port3);
-
- ServiceManager manager = new ServiceManager(haContext);
- ServiceInstance<Void> instance = manager.getServiceInstance();
- assertEquals("'instance' doesn't match.", instance.getPort(), servicesMap.get(instance.getAddress()));
- instance = manager.getServiceInstance();
- assertEquals("'instance' doesn't match.", instance.getPort(), servicesMap.get(instance.getAddress()));
- instance = manager.getServiceInstance();
- assertEquals("'instance' doesn't match.", instance.getPort(), servicesMap.get(instance.getAddress()));
- }
-
- @Test
- public void testReportError() throws Exception {
- final String hostname1 = "localhost1";
- final Integer port1 = 123;
-
- ServiceRegister register1 = new ServiceRegister(haContext);
- register1.regService(hostname1, port1);
-
- ServiceManager manager = new ServiceManager(haContext);
- ServiceInstance<Void> instance = manager.getServiceInstance();
- manager.reportError(instance);
- // report twice, manager will not return temporarily
- instance = manager.getServiceInstance();
- manager.reportError(instance);
- instance = manager.getServiceInstance();
- assertEquals("'instance' should be null.", null, instance);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/sentry/blob/5630fc5c/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/thrift/TestSentryServerForHaWithoutKerberos.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/thrift/TestSentryServerForHaWithoutKerberos.java b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/thrift/TestSentryServerForHaWithoutKerberos.java
deleted file mode 100644
index 6c78942..0000000
--- a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/thrift/TestSentryServerForHaWithoutKerberos.java
+++ /dev/null
@@ -1,219 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless createRequired by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.sentry.provider.db.service.thrift;
-import static org.junit.Assert.assertEquals;
-
-import java.util.HashSet;
-import java.util.Set;
-
-import org.apache.sentry.core.common.ActiveRoleSet;
-import org.apache.sentry.core.model.db.AccessConstants;
-import org.apache.sentry.core.model.db.Database;
-import org.apache.sentry.core.model.db.Server;
-import org.apache.sentry.core.model.db.Table;
-import org.apache.sentry.service.thrift.SentryServiceIntegrationBase;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-
-public class TestSentryServerForHaWithoutKerberos extends SentryServiceIntegrationBase {
-
- @BeforeClass
- public static void setup() throws Exception {
- kerberos = false;
- haEnabled = true;
- beforeSetup();
- setupConf();
- startSentryService();
- afterSetup();
- }
-
- @Test
- public void testCreateRole() throws Exception {
- String requestorUserName = ADMIN_USER;
- Set<String> requestorUserGroupNames = Sets.newHashSet(ADMIN_GROUP);
- setLocalGroupMapping(requestorUserName, requestorUserGroupNames);
- writePolicyFile();
- String roleName = "admin_r";
- client.dropRoleIfExists(requestorUserName, roleName);
- client.createRole(requestorUserName, roleName);
- client.dropRole(requestorUserName, roleName);
- }
-
- @Test
- public void testQueryPushDown() throws Exception {
- String requestorUserName = ADMIN_USER;
- Set<String> requestorUserGroupNames = Sets.newHashSet(ADMIN_GROUP);
- setLocalGroupMapping(requestorUserName, requestorUserGroupNames);
- writePolicyFile();
-
- String roleName1 = "admin_r1";
- String roleName2 = "admin_r2";
-
- String group1 = "g1";
- String group2 = "g2";
-
- client.dropRoleIfExists(requestorUserName, roleName1);
- client.createRole(requestorUserName, roleName1);
- client.grantRoleToGroup(requestorUserName, group1, roleName1);
-
- client.grantTablePrivilege(requestorUserName, roleName1, "server", "db1", "table1", "ALL");
- client.grantTablePrivilege(requestorUserName, roleName1, "server", "db1", "table2", "ALL");
- client.grantTablePrivilege(requestorUserName, roleName1, "server", "db2", "table3", "ALL");
- client.grantTablePrivilege(requestorUserName, roleName1, "server", "db2", "table4", "ALL");
-
-
- client.dropRoleIfExists(requestorUserName, roleName2);
- client.createRole(requestorUserName, roleName2);
- client.grantRoleToGroup(requestorUserName, group1, roleName2);
- client.grantRoleToGroup(requestorUserName, group2, roleName2);
-
- client.grantTablePrivilege(requestorUserName, roleName2, "server", "db1", "table1", "ALL");
- client.grantTablePrivilege(requestorUserName, roleName2, "server", "db1", "table2", "ALL");
- client.grantTablePrivilege(requestorUserName, roleName2, "server", "db2", "table3", "ALL");
- client.grantTablePrivilege(requestorUserName, roleName2, "server", "db2", "table4", "ALL");
- client.grantTablePrivilege(requestorUserName, roleName2, "server", "db3", "table5", "ALL");
-
- Set<TSentryPrivilege> listPrivilegesByRoleName = client.listPrivilegesByRoleName(requestorUserName, roleName2, Lists.newArrayList(new Server("server"), new Database("db1")));
- assertEquals("Privilege not assigned to role2 !!", 2, listPrivilegesByRoleName.size());
-
- listPrivilegesByRoleName = client.listPrivilegesByRoleName(requestorUserName, roleName2, Lists.newArrayList(new Server("server"), new Database("db2"), new Table("table1")));
- assertEquals("Privilege not assigned to role2 !!", 0, listPrivilegesByRoleName.size());
-
- listPrivilegesByRoleName = client.listPrivilegesByRoleName(requestorUserName, roleName2, Lists.newArrayList(new Server("server"), new Database("db1"), new Table("table1")));
- assertEquals("Privilege not assigned to role2 !!", 1, listPrivilegesByRoleName.size());
-
- listPrivilegesByRoleName = client.listPrivilegesByRoleName(requestorUserName, roleName2, Lists.newArrayList(new Server("server"), new Database("db3")));
- assertEquals("Privilege not assigned to role2 !!", 1, listPrivilegesByRoleName.size());
-
- Set<String> listPrivilegesForProvider = client.listPrivilegesForProvider(Sets.newHashSet(group1, group2), null, ActiveRoleSet.ALL, new Server("server"), new Database("db2"));
- assertEquals("Privilege not correctly assigned to roles !!",
- Sets.newHashSet("server=server->db=db2->table=table4->action=all", "server=server->db=db2->table=table3->action=all"),
- listPrivilegesForProvider);
-
- listPrivilegesForProvider = client.listPrivilegesForProvider(Sets.newHashSet(group1, group2), null, ActiveRoleSet.ALL, new Server("server"), new Database("db3"));
- assertEquals("Privilege not correctly assigned to roles !!", Sets.newHashSet("server=server->db=db3->table=table5->action=all"), listPrivilegesForProvider);
-
- listPrivilegesForProvider = client.listPrivilegesForProvider(Sets.newHashSet(group1, group2), null, new ActiveRoleSet(Sets.newHashSet(roleName1)), new Server("server"), new Database("db3"));
- assertEquals("Privilege not correctly assigned to roles !!", Sets.newHashSet("server=+"), listPrivilegesForProvider);
-
- listPrivilegesForProvider = client.listPrivilegesForProvider(Sets.newHashSet(group1, group2), null, new ActiveRoleSet(Sets.newHashSet(roleName1)), new Server("server1"));
- assertEquals("Privilege not correctly assigned to roles !!", new HashSet<String>(), listPrivilegesForProvider);
- }
-
-
-
- /**
- * Create role, add privileges and grant it to a group drop the role and
- * verify the privileges are no longer visible recreate the role with same
- * name and verify the privileges again.
- * @throws Exception
- */
- @Test
- public void testDropRole() throws Exception {
- String requestorUserName = ADMIN_USER;
- Set<String> requestorUserGroupNames = Sets.newHashSet(ADMIN_GROUP);
- setLocalGroupMapping(requestorUserName, requestorUserGroupNames);
- writePolicyFile();
- String roleName = "admin_r";
-
- // create role and add privileges
- client.dropRoleIfExists(requestorUserName, roleName);
- client.createRole(requestorUserName, roleName);
- client.grantRoleToGroup(requestorUserName, ADMIN_GROUP, roleName);
- client.grantDatabasePrivilege(requestorUserName, roleName, "server1", "db2", AccessConstants.ALL);
- client.grantTablePrivilege(requestorUserName, roleName, "server1", "db3", "tab3", "ALL");
- assertEquals(2, client.listPrivilegesForProvider(requestorUserGroupNames, null,
- ActiveRoleSet.ALL).size());
-
- // drop role and verify privileges
- client.dropRole(requestorUserName, roleName);
- assertEquals(0, client.listPrivilegesForProvider(requestorUserGroupNames, null,
- ActiveRoleSet.ALL).size());
-
- // recreate the role
- client.createRole(requestorUserName, roleName);
- client.grantRoleToGroup(requestorUserName, ADMIN_GROUP, roleName);
- assertEquals(0, client.listPrivilegesForProvider(requestorUserGroupNames, null,
- ActiveRoleSet.ALL).size());
-
- // grant different privileges and verify
- client.grantDatabasePrivilege(requestorUserName, roleName, "server1", "db2", AccessConstants.ALL);
- assertEquals(1, client.listPrivilegesForProvider(requestorUserGroupNames, null,
- ActiveRoleSet.ALL).size());
- client.dropRole(requestorUserName, roleName);
- assertEquals(0, client.listPrivilegesForProvider(requestorUserGroupNames, null,
- ActiveRoleSet.ALL).size());
- assertEquals(0, client.listPrivilegesForProvider(requestorUserGroupNames, null,
- ActiveRoleSet.ALL).size());
- }
-
- @Test
- public void testDropRoleOnUser() throws Exception {
- String requestorUserName = ADMIN_USER;
- Set<String> requestorUserGroupNames = Sets.newHashSet(ADMIN_GROUP);
- Set<String> requestorUserNames = Sets.newHashSet(ADMIN_USER);
- setLocalGroupMapping(requestorUserName, requestorUserGroupNames);
- writePolicyFile();
- String roleName = "admin_r";
-
- // create role and add privileges
- client.dropRoleIfExists(requestorUserName, roleName);
- client.createRole(requestorUserName, roleName);
- client.grantRoleToUser(requestorUserName, ADMIN_USER, roleName);
- client.grantDatabasePrivilege(requestorUserName, roleName, "server1", "db2", AccessConstants.ALL);
- client.grantTablePrivilege(requestorUserName, roleName, "server1", "db3", "tab3", "ALL");
- assertEquals(2, client.listPrivilegesForProvider(requestorUserGroupNames, requestorUserNames,
- ActiveRoleSet.ALL).size());
-
- // drop role and verify privileges
- client.dropRole(requestorUserName, roleName);
- assertEquals(0, client.listPrivilegesForProvider(requestorUserGroupNames, requestorUserNames,
- ActiveRoleSet.ALL).size());
-
- // recreate the role
- client.createRole(requestorUserName, roleName);
- client.grantRoleToGroup(requestorUserName, ADMIN_GROUP, roleName);
- assertEquals(0, client.listPrivilegesForProvider(requestorUserGroupNames, requestorUserNames,
- ActiveRoleSet.ALL).size());
-
- // grant different privileges and verify
- client.grantDatabasePrivilege(requestorUserName, roleName, "server1", "db2", AccessConstants.ALL);
- assertEquals(1, client.listPrivilegesForProvider(requestorUserGroupNames, requestorUserNames,
- ActiveRoleSet.ALL).size());
- client.dropRole(requestorUserName, roleName);
- assertEquals(0, client.listPrivilegesForProvider(requestorUserGroupNames, requestorUserNames,
- ActiveRoleSet.ALL).size());
- assertEquals(0, client.listPrivilegesForProvider(requestorUserGroupNames, requestorUserNames,
- ActiveRoleSet.ALL).size());
- }
-
- /**
- * Test that we are correctly substituting "_HOST" if/when needed.
- *
- * @throws Exception
- */
- @Test
- public void testHostSubstitution() throws Exception {
- // We just need to ensure that we are able to correct connect to the server
- connectToSentryService();
- }
-}
http://git-wip-us.apache.org/repos/asf/sentry/blob/5630fc5c/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/thrift/TestSentryServerForPoolHAWithoutKerberos.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/thrift/TestSentryServerForPoolHAWithoutKerberos.java b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/thrift/TestSentryServerForPoolHAWithoutKerberos.java
deleted file mode 100644
index 9ba7d23..0000000
--- a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/thrift/TestSentryServerForPoolHAWithoutKerberos.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless createRequired by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.sentry.provider.db.service.thrift;
-
-import org.junit.BeforeClass;
-
-public class TestSentryServerForPoolHAWithoutKerberos extends TestSentryServerForHaWithoutKerberos {
-
- @BeforeClass
- public static void setup() throws Exception {
- kerberos = false;
- haEnabled = true;
- pooled = true;
- beforeSetup();
- setupConf();
- startSentryService();
- afterSetup();
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/sentry/blob/5630fc5c/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/thrift/TestSentryServiceForHAWithKerberos.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/thrift/TestSentryServiceForHAWithKerberos.java b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/thrift/TestSentryServiceForHAWithKerberos.java
deleted file mode 100644
index 813b30b..0000000
--- a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/thrift/TestSentryServiceForHAWithKerberos.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sentry.provider.db.service.thrift;
-
-
-import java.io.File;
-import java.util.Set;
-
-import org.apache.sentry.provider.file.PolicyFile;
-import org.apache.sentry.service.thrift.SentryServiceIntegrationBase;
-import org.apache.sentry.service.thrift.ServiceConstants.ServerConfig;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import com.google.common.collect.Sets;
-
-/**
- * Test various kerberos related stuff on the SentryService side
- */
-public class TestSentryServiceForHAWithKerberos extends SentryServiceIntegrationBase {
-
- @BeforeClass
- public static void setup() throws Exception {
- kerberos = true;
- haEnabled = true;
- SERVER_KERBEROS_NAME = "sentry/_HOST@" + REALM;
- beforeSetup();
- setupConf();
- startSentryService();
- afterSetup();
- }
-
- @Override
- @Before
- public void before() throws Exception {
- policyFilePath = new File(dbDir, "local_policy_file.ini");
- conf.set(ServerConfig.SENTRY_STORE_GROUP_MAPPING_RESOURCE,
- policyFilePath.getPath());
- policyFile = new PolicyFile();
- connectToSentryService();
- }
-
- @Test
- public void testCreateRole() throws Exception {
- runTestAsSubject(new TestOperation(){
- @Override
- public void runTestAsSubject() throws Exception {
- String requestorUserName = ADMIN_USER;
- Set<String> requestorUserGroupNames = Sets.newHashSet(ADMIN_GROUP);
- setLocalGroupMapping(requestorUserName, requestorUserGroupNames);
- writePolicyFile();
- String roleName = "admin_r";
- client.dropRoleIfExists(requestorUserName, roleName);
- client.createRole(requestorUserName, roleName);
- client.dropRole(requestorUserName, roleName);
- }
- });
- }
-}
http://git-wip-us.apache.org/repos/asf/sentry/blob/5630fc5c/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/thrift/TestSentryServiceForPoolHAWithKerberos.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/thrift/TestSentryServiceForPoolHAWithKerberos.java b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/thrift/TestSentryServiceForPoolHAWithKerberos.java
deleted file mode 100644
index acb906f..0000000
--- a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/thrift/TestSentryServiceForPoolHAWithKerberos.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless createRequired by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.sentry.provider.db.service.thrift;
-
-import org.junit.BeforeClass;
-
-public class TestSentryServiceForPoolHAWithKerberos extends TestSentryServiceWithKerberos {
-
- @BeforeClass
- public static void setup() throws Exception {
- kerberos = true;
- haEnabled = true;
- pooled = true;
- beforeSetup();
- setupConf();
- startSentryService();
- afterSetup();
- }
-
-}
\ No newline at end of file