You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sentry.apache.org by va...@apache.org on 2017/06/14 00:56:50 UTC
[12/52] [abbrv] sentry git commit: SENTRY-1649: Initialize
HMSFollower when sentry server actually starts (Na Li,
reviewed by: Alex Kolbasov)
SENTRY-1649: Initialize HMSFollower when sentry server actually starts (Na Li, reviewed by: Alex Kolbasov)
Conflicts:
sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceUtil.java
Change-Id: Ic2eda3cd800e7d6e2febd2cebbacf190663ce288
Reviewed-on: http://gerrit.sjc.cloudera.com:8080/22402
Tested-by: Jenkins User
Reviewed-by: Alexander Kolbasov <ak...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/sentry/repo
Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/5bdecd78
Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/5bdecd78
Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/5bdecd78
Branch: refs/for/cdh5-1.5.1_ha
Commit: 5bdecd78c6bcf0c1151d99d8621eef258b1dc677
Parents: afc1d82
Author: Alexander Kolbasov <ak...@cloudera.com>
Authored: Fri May 5 20:22:29 2017 -0700
Committer: Alexander Kolbasov <ak...@cloudera.com>
Committed: Mon May 8 14:42:13 2017 -0700
----------------------------------------------------------------------
.../sentry/service/thrift/HMSFollower.java | 24 ++-
.../sentry/service/thrift/SentryService.java | 168 +++++++++++++------
.../service/thrift/SentryServiceUtil.java | 70 ++++++++
.../sentry/service/thrift/ServiceConstants.java | 2 +-
.../TestDbSentryOnFailureHookLoading.java | 10 +-
.../tests/e2e/hdfs/TestHDFSIntegrationBase.java | 7 +-
.../e2e/hdfs/TestHDFSIntegrationEnd2End.java | 20 ++-
7 files changed, 235 insertions(+), 66 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sentry/blob/5bdecd78/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java
index c3f821c..da576c9 100644
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java
@@ -68,10 +68,11 @@ import static org.apache.sentry.hdfs.Updateable.Update;
* stored for HDFS-Sentry sync.
*/
@SuppressWarnings("PMD")
-public class HMSFollower implements Runnable {
+public class HMSFollower implements Runnable, AutoCloseable {
private static final Logger LOGGER = LoggerFactory.getLogger(HMSFollower.class);
-
private long currentEventID;
+ // Copy from Hive
+ public static String CONF_METASTORE_URI = "hive.metastore.uris";
// Track the latest eventId of the event that has been logged. So we don't log the same message
private long lastLoggedEventId = SentryStore.EMPTY_CHANGE_ID;
private static boolean connectedToHMS = false;
@@ -85,8 +86,7 @@ public class HMSFollower implements Runnable {
private boolean needHiveSnapshot = true;
private final LeaderStatusMonitor leaderMonitor;
- HMSFollower(Configuration conf, SentryStore store, LeaderStatusMonitor leaderMonitor)
- throws Exception {
+ HMSFollower(Configuration conf, SentryStore store, LeaderStatusMonitor leaderMonitor) throws Exception {
LOGGER.info("HMSFollower is being initialized");
authzConf = conf;
this.leaderMonitor = leaderMonitor;
@@ -111,6 +111,12 @@ public class HMSFollower implements Runnable {
return connectedToHMS;
}
+ @Override
+ public void close() {
+ // Close any outstanding connections to HMS
+ closeHMSConnection();
+ }
+
/**
* Returns HMS Client if successful, returns null if HMS is not ready yet to take connections
* Throws @LoginException if Kerberos context creation failed using Sentry's kerberos credentials
@@ -122,6 +128,12 @@ public class HMSFollower implements Runnable {
return client;
}
+ // Do not create client if the metastre URI in the configuration is missing
+ if (conf.get(CONF_METASTORE_URI, "").isEmpty()) {
+ // Come back later with real Hive URI
+ return null;
+ }
+
final HiveConf hiveConf = new HiveConf();
hiveInstance = hiveConf.get(HiveAuthzConf.AuthzConfVars.AUTHZ_SERVER_NAME.getVar());
@@ -149,7 +161,7 @@ public class HMSFollower implements Runnable {
"Kerberos principal should have 3 parts: " + principal);
keytab = Preconditions.checkNotNull(conf.get(ServiceConstants.ServerConfig.KEY_TAB),
- ServiceConstants.ServerConfig.KEY_TAB + " is required");
+ ServiceConstants.ServerConfig.KEY_TAB + " is required");
File keytabFile = new File(keytab);
Preconditions.checkState(keytabFile.isFile() && keytabFile.canRead(),
"Keytab " + keytab + " does not exist or is not readable.");
@@ -560,7 +572,7 @@ public class HMSFollower implements Runnable {
} catch (SentryNoSuchObjectException e) {
LOGGER.info("Drop Sentry privilege ignored as there are no privileges on the table: %s.%s", dbName, tableName);
} catch (Exception e) {
- throw new SentryInvalidInputException("Could not process Create table event. Event: " + event.toString(), e);
+ throw new SentryInvalidInputException("Could not process Drop table event. Event: " + event.toString(), e);
}
}
http://git-wip-us.apache.org/repos/asf/sentry/blob/5bdecd78/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java
index 1cfc7be..a8c8e05 100644
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java
@@ -70,6 +70,7 @@ import com.google.common.base.Preconditions;
import static org.apache.sentry.core.common.utils.SigUtils.registerSigListener;
+// Enable signal handler for HA leader/follower status if configured
public class SentryService implements Callable, SigUtils.SigListener {
private static final Logger LOGGER = LoggerFactory.getLogger(SentryService.class);
@@ -89,6 +90,7 @@ public class SentryService implements Callable, SigUtils.SigListener {
private final String keytab;
private final ExecutorService serviceExecutor;
private ScheduledExecutorService hmsFollowerExecutor = null;
+ private HMSFollower hmsFollower = null;
private Future serviceStatus;
private TServer thriftServer;
private Status status;
@@ -101,8 +103,7 @@ public class SentryService implements Callable, SigUtils.SigListener {
{@link HMSFollower}.
*/
private final SentryStore sentryStore;
- private final ScheduledExecutorService sentryStoreCleanService =
- Executors.newSingleThreadScheduledExecutor();
+ private ScheduledExecutorService sentryStoreCleanService;
private final LeaderStatusMonitor leaderMonitor;
private final boolean notificationLogEnabled;
@@ -164,21 +165,6 @@ public class SentryService implements Callable, SigUtils.SigListener {
notificationLogEnabled = conf.getBoolean(ServerConfig.SENTRY_NOTIFICATION_LOG_ENABLED,
ServerConfig.SENTRY_NOTIFICATION_LOG_ENABLED_DEFAULT);
- if (notificationLogEnabled) {
- try {
- long initDelay = conf.getLong(ServerConfig.SENTRY_HMSFOLLOWER_INIT_DELAY_MILLS,
- ServerConfig.SENTRY_HMSFOLLOWER_INIT_DELAY_MILLS_DEFAULT);
- long period = conf.getLong(ServerConfig.SENTRY_HMSFOLLOWER_INTERVAL_MILLS,
- ServerConfig.SENTRY_HMSFOLLOWER_INTERVAL_MILLS_DEFAULT);
- hmsFollowerExecutor = Executors.newScheduledThreadPool(1);
- hmsFollowerExecutor.scheduleAtFixedRate(new HMSFollower(conf, sentryStore, leaderMonitor),
- initDelay, period, TimeUnit.MILLISECONDS);
- } catch (Exception e) {
- //TODO: Handle
- LOGGER.error("Could not start HMSFollower");
- }
- }
-
status = Status.NOT_STARTED;
// Enable signal handler for HA leader/follower status if configured
@@ -191,25 +177,6 @@ public class SentryService implements Callable, SigUtils.SigListener {
LOGGER.error("Failed to register signal", e);
}
}
-
- // If SENTRY_STORE_CLEAN_PERIOD_SECONDS is set to positive, the background SentryStore cleaning
- // thread is enabled. Currently, it only purges the delta changes {@link MSentryChange} in
- // the sentry store.
- long storeCleanPeriodSecs = conf.getLong(
- ServerConfig.SENTRY_STORE_CLEAN_PERIOD_SECONDS,
- ServerConfig.SENTRY_STORE_CLEAN_PERIOD_SECONDS_DEFAULT);
- if (storeCleanPeriodSecs > 0) {
- Runnable storeCleaner = new Runnable() {
- @Override
- public void run() {
- if (leaderMonitor.isLeader()) {
- sentryStore.purgeDeltaChangeTables();
- }
- }
- };
- sentryStoreCleanService.scheduleWithFixedDelay(
- storeCleaner, 0, storeCleanPeriodSecs, TimeUnit.SECONDS);
- }
}
@Override
@@ -242,6 +209,10 @@ public class SentryService implements Callable, SigUtils.SigListener {
}
private void runServer() throws Exception {
+
+ startSentryStoreCleaner(conf);
+ startHMSFollower(conf);
+
Iterable<String> processorFactories = ConfUtilties.CLASS_SPLITTER
.split(conf.get(ServerConfig.PROCESSOR_FACTORIES,
ServerConfig.PROCESSOR_FACTORIES_DEFAULT).trim());
@@ -281,7 +252,7 @@ public class SentryService implements Callable, SigUtils.SigListener {
TSaslServerTransport.Factory saslTransportFactory = new TSaslServerTransport.Factory();
saslTransportFactory.addServerDefinition(AuthMethod.KERBEROS
.getMechanismName(), principalParts[0], principalParts[1],
- ServerConfig.SASL_PROPERTIES, new GSSCallback(conf));
+ ServerConfig.SASL_PROPERTIES, new GSSCallback(conf));
transportFactory = saslTransportFactory;
} else {
transportFactory = new TTransportFactory();
@@ -297,6 +268,110 @@ public class SentryService implements Callable, SigUtils.SigListener {
thriftServer.serve();
}
+ private void startHMSFollower(Configuration conf) throws Exception{
+ if (!notificationLogEnabled) {
+ return;
+ }
+
+ Preconditions.checkState(hmsFollower == null);
+ Preconditions.checkState(hmsFollowerExecutor == null);
+
+ try {
+ hmsFollower = new HMSFollower(conf, sentryStore, leaderMonitor);
+ } catch (Exception ex) {
+ LOGGER.error("Could not create HMSFollower", ex);
+ throw ex;
+ }
+
+ long initDelay = conf.getLong(ServerConfig.SENTRY_HMSFOLLOWER_INIT_DELAY_MILLS,
+ ServerConfig.SENTRY_HMSFOLLOWER_INIT_DELAY_MILLS_DEFAULT);
+ long period = conf.getLong(ServerConfig.SENTRY_HMSFOLLOWER_INTERVAL_MILLS,
+ ServerConfig.SENTRY_HMSFOLLOWER_INTERVAL_MILLS_DEFAULT);
+ try {
+ hmsFollowerExecutor = Executors.newScheduledThreadPool(1);
+ hmsFollowerExecutor.scheduleAtFixedRate(hmsFollower,
+ initDelay, period, TimeUnit.MILLISECONDS);
+ } catch (IllegalArgumentException e) {
+ LOGGER.error(String.format("Could not start HMSFollower due to illegal argument. period is %s ms", period), e);
+ throw e;
+ }
+ }
+
+ private void stopHMSFollower(Configuration conf) {
+ if (!notificationLogEnabled) {
+ return;
+ }
+
+ Preconditions.checkNotNull(hmsFollowerExecutor);
+ Preconditions.checkNotNull(hmsFollower);
+
+ // use follower scheduling interval as timeout for shutting down its executor as
+ // such scheduling interval should be an upper bound of how long the task normally takes to finish
+ long timeoutValue = conf.getLong(ServerConfig.SENTRY_HMSFOLLOWER_INTERVAL_MILLS,
+ ServerConfig.SENTRY_HMSFOLLOWER_INTERVAL_MILLS_DEFAULT);
+ try {
+ SentryServiceUtil.shutdownAndAwaitTermination(hmsFollowerExecutor, "hmsFollowerExecutor",
+ timeoutValue, TimeUnit.MILLISECONDS, LOGGER);
+ } finally {
+ hmsFollowerExecutor = null;
+ try {
+ // close connections
+ hmsFollower.close();
+ } catch (RuntimeException ex) {
+ LOGGER.error("HMSFollower.close() failed", ex);
+ } finally {
+ hmsFollower = null;
+ }
+ }
+ }
+
+ private void startSentryStoreCleaner(Configuration conf) {
+ Preconditions.checkState(sentryStoreCleanService == null);
+
+ // If SENTRY_STORE_CLEAN_PERIOD_SECONDS is set to positive, the background SentryStore cleaning
+ // thread is enabled. Currently, it only purges the delta changes {@link MSentryChange} in
+ // the sentry store.
+ long storeCleanPeriodSecs = conf.getLong(
+ ServerConfig.SENTRY_STORE_CLEAN_PERIOD_SECONDS,
+ ServerConfig.SENTRY_STORE_CLEAN_PERIOD_SECONDS_DEFAULT);
+ if (storeCleanPeriodSecs <= 0) {
+ return;
+ }
+
+ try {
+ Runnable storeCleaner = new Runnable() {
+ @Override
+ public void run() {
+ if (leaderMonitor.isLeader()) {
+ sentryStore.purgeDeltaChangeTables();
+ }
+ }
+ };
+
+ sentryStoreCleanService = Executors.newSingleThreadScheduledExecutor();
+ sentryStoreCleanService.scheduleWithFixedDelay(
+ storeCleaner, 0, storeCleanPeriodSecs, TimeUnit.SECONDS);
+
+ LOGGER.info("sentry store cleaner is scheduled with interval %d seconds", storeCleanPeriodSecs);
+ }
+ catch(IllegalArgumentException e){
+ LOGGER.error("Could not start SentryStoreCleaner due to illegal argument", e);
+ sentryStoreCleanService = null;
+ }
+ }
+
+ private void stopSentryStoreCleaner() {
+ Preconditions.checkNotNull(sentryStoreCleanService);
+
+ try {
+ SentryServiceUtil.shutdownAndAwaitTermination(sentryStoreCleanService, "sentryStoreCleanService",
+ 10, TimeUnit.SECONDS, LOGGER);
+ }
+ finally {
+ sentryStoreCleanService = null;
+ }
+ }
+
private void addSentryServiceGauge() {
SentryMetrics.getInstance().addSentryServiceGauges(this);
}
@@ -311,7 +386,6 @@ public class SentryService implements Callable, SigUtils.SigListener {
sentryWebServer = new SentryWebServer(listenerList, webServerPort, conf);
sentryWebServer.start();
}
-
}
private void stopSentryWebServer() throws Exception{
@@ -368,20 +442,10 @@ public class SentryService implements Callable, SigUtils.SigListener {
} else {
LOGGER.info("Sentry web service is already stopped...");
}
- if(hmsFollowerExecutor != null) {
- hmsFollowerExecutor.shutdown();
- }
- sentryStoreCleanService.shutdown();
- try {
- if (!sentryStoreCleanService.awaitTermination(10, TimeUnit.SECONDS)) {
- sentryStoreCleanService.shutdownNow();
- if (!sentryStoreCleanService.awaitTermination(10, TimeUnit.SECONDS)) {
- LOGGER.error("DeltaCleanerService did not terminate");
- }
- }
- } catch (InterruptedException ie) {
- sentryStoreCleanService.shutdownNow();
- }
+
+ stopHMSFollower(conf);
+ stopSentryStoreCleaner();
+
if (exception != null) {
exception.ifExceptionThrow();
}
http://git-wip-us.apache.org/repos/asf/sentry/blob/5bdecd78/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceUtil.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceUtil.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceUtil.java
new file mode 100644
index 0000000..d58ee95
--- /dev/null
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceUtil.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.service.thrift;
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+
+public final class SentryServiceUtil {
+ /**
+ * Gracefully shut down an Executor service.
+ * <p>
+ * This code is based on the Javadoc example for the Executor service.
+ * <p>
+ * First call shutdown to reject incoming tasks, and then call
+ * shutdownNow, if necessary, to cancel any lingering tasks.
+ *
+ * @param pool the executor service to shut down
+ * @param poolName the name of the executor service to shut down to make it easy for debugging
+ * @param timeout the timeout interval to wait for its termination
+ * @param unit the unit of the timeout
+ * @param logger the logger to log the error message if it cannot terminate. It could be null
+ */
+ static void shutdownAndAwaitTermination(ExecutorService pool, String poolName,
+ long timeout, TimeUnit unit, Logger logger) {
+ Preconditions.checkNotNull(pool);
+
+ pool.shutdown(); // Disable new tasks from being submitted
+ try {
+ // Wait a while for existing tasks to terminate
+ if (!pool.awaitTermination(timeout, unit)) {
+ pool.shutdownNow(); // Cancel currently executing tasks
+ // Wait a while for tasks to respond to being cancelled
+ if ((!pool.awaitTermination(timeout, unit)) && (logger != null)) {
+ logger.error("Executor service {} did not terminate",
+ StringUtils.defaultIfBlank(poolName, "null"));
+ }
+ }
+ } catch (InterruptedException ignored) {
+ // (Re-)Cancel if current thread also interrupted
+ pool.shutdownNow();
+ // Preserve interrupt status
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ private SentryServiceUtil() {
+ // Make constructor private to avoid instantiation
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/sentry/blob/5bdecd78/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/ServiceConstants.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/ServiceConstants.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/ServiceConstants.java
index 29fb421..c14a854 100644
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/ServiceConstants.java
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/ServiceConstants.java
@@ -168,7 +168,7 @@ public class ServiceConstants {
// InitialDelay and period time for HMSFollower thread.
public static final String SENTRY_HMSFOLLOWER_INIT_DELAY_MILLS = "sentry.hmsfollower.init.delay.mills";
- public static final long SENTRY_HMSFOLLOWER_INIT_DELAY_MILLS_DEFAULT = 60000;
+ public static final long SENTRY_HMSFOLLOWER_INIT_DELAY_MILLS_DEFAULT = 0;
public static final String SENTRY_HMSFOLLOWER_INTERVAL_MILLS = "sentry.hmsfollower.interval.mills";
public static final long SENTRY_HMSFOLLOWER_INTERVAL_MILLS_DEFAULT = 500;
http://git-wip-us.apache.org/repos/asf/sentry/blob/5bdecd78/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbSentryOnFailureHookLoading.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbSentryOnFailureHookLoading.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbSentryOnFailureHookLoading.java
index b06cf59..05de07d 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbSentryOnFailureHookLoading.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbSentryOnFailureHookLoading.java
@@ -39,9 +39,12 @@ import org.junit.After;
import org.junit.Assume;
import org.junit.BeforeClass;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class TestDbSentryOnFailureHookLoading extends AbstractTestWithDbProvider {
+ private static final Logger LOGGER = LoggerFactory.getLogger(TestDbSentryOnFailureHookLoading.class);
@BeforeClass
public static void setup() throws Exception {
@@ -267,7 +270,12 @@ public class TestDbSentryOnFailureHookLoading extends AbstractTestWithDbProvider
}
if(dbName != null) {
Assert.assertNotNull("Database object is null for op: " + expectedOp, DummySentryOnFailureHook.db);
- Assert.assertTrue(dbName.equalsIgnoreCase(DummySentryOnFailureHook.db.getName()));
+
+ String failureHookDbName = DummySentryOnFailureHook.db.getName();
+ LOGGER.debug("dbName: {}; DummySentryOnFailureHook.db.getName(): {}", dbName, failureHookDbName);
+ if(!failureHookDbName.equalsIgnoreCase("*")) {
+ Assert.assertTrue(dbName.equalsIgnoreCase(failureHookDbName));
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/sentry/blob/5bdecd78/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationBase.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationBase.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationBase.java
index 01f1ec7..eb290a9 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationBase.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationBase.java
@@ -148,8 +148,11 @@ public abstract class TestHDFSIntegrationBase {
protected static boolean testSentryHA = false;
protected boolean ignoreCleanUp = false;
protected static final long STALE_THRESHOLD = 5000;
- protected static final long CACHE_REFRESH = 100; //Default is 500, but we want it to be low
- // in our tests so that changes reflect soon
+
+ // we want to make sure the cache is updated in our tests so that changes reflect soon
+ protected static final long CACHE_REFRESH =
+ ServerConfig.SENTRY_HMSFOLLOWER_INIT_DELAY_MILLS_DEFAULT +
+ ServerConfig.SENTRY_HMSFOLLOWER_INTERVAL_MILLS_DEFAULT * 2;
protected static String fsURI;
protected static int hmsPort;
http://git-wip-us.apache.org/repos/asf/sentry/blob/5bdecd78/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationEnd2End.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationEnd2End.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationEnd2End.java
index 884ccbe..b7a9109 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationEnd2End.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationEnd2End.java
@@ -41,6 +41,8 @@ public class TestHDFSIntegrationEnd2End extends TestHDFSIntegrationBase {
private static final Logger LOGGER = LoggerFactory
.getLogger(TestHDFSIntegrationEnd2End.class);
+ private static String adminRole = "admin_role";
+
@Ignore
@Test
public void testEnd2End() throws Throwable {
@@ -371,8 +373,10 @@ public class TestHDFSIntegrationEnd2End extends TestHDFSIntegrationBase {
}
//SENTRY-780
+ @Ignore // SENTRY-1750 HMSFollower does not handle view update correctly. Enable this test once SENTRY-1750 is fixed
@Test
public void testViews() throws Throwable {
+ LOGGER.info("testViews starts");
String dbName= "db1";
tmpHDFSDir = new Path("/tmp/external");
@@ -404,6 +408,7 @@ public class TestHDFSIntegrationEnd2End extends TestHDFSIntegrationBase {
stmt.close();
conn.close();
+ LOGGER.info("testViews ends");
}
/*
@@ -411,11 +416,13 @@ TODO:SENTRY-819
*/
@Test
public void testAllColumn() throws Throwable {
+ LOGGER.info("testAllColumn starts");
String dbName = "db2";
+ String userRole = "col1_role";
tmpHDFSDir = new Path("/tmp/external");
dbNames = new String[]{dbName};
- roles = new String[]{"admin_role", "col_role"};
+ roles = new String[]{"admin_role", userRole};
admin = StaticUserGroup.ADMIN1;
Connection conn;
@@ -435,9 +442,9 @@ TODO:SENTRY-819
stmt.execute("alter table p1 add partition (month=1, day=1)");
loadDataTwoCols(stmt);
- stmt.execute("create role col_role");
- stmt.execute("grant select(c1,c2) on p1 to role col_role");
- stmt.execute("grant role col_role to group "+ StaticUserGroup.USERGROUP1);
+ stmt.execute("create role " + userRole);
+ stmt.execute("grant select(c1,c2) on p1 to role " + userRole);
+ stmt.execute("grant role " + userRole + " to group "+ StaticUserGroup.USERGROUP1);
Thread.sleep(100);
//User with privileges on all columns of the data cannot still read the HDFS files
@@ -445,15 +452,19 @@ TODO:SENTRY-819
stmt.close();
conn.close();
+ LOGGER.info("testAllColumn ends");
}
@Test
public void testColumnPrivileges() throws Throwable {
+ LOGGER.info("testColumnPrivileges starts");
String dbName = "db2";
tmpHDFSDir = new Path("/tmp/external");
dbNames = new String[]{dbName};
+ dbNames = new String[]{dbName};
roles = new String[]{"admin_role", "tab_role", "db_role", "col_role"};
+ roles = new String[]{adminRole, "tab_role", "db_role", "col_role"};
admin = StaticUserGroup.ADMIN1;
Connection conn;
@@ -510,6 +521,7 @@ TODO:SENTRY-819
stmt.close();
conn.close();
+ LOGGER.info("testColumnPrivileges ends");
}