You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sentry.apache.org by ha...@apache.org on 2017/01/20 19:33:18 UTC
sentry git commit: SENTRY-1428: Only leader should follow HMS updates
(Alexander Kolbasov, Reviewed by: Hao Hao)
Repository: sentry
Updated Branches:
refs/heads/sentry-ha-redesign 79cbd178a -> c4d3b5eae
SENTRY-1428: Only leader should follow HMS updates (Alexander Kolbasov, Reviewed by: Hao Hao)
Change-Id: I158608d917241fe01ad7a12b2c4ad8e633b6a23d
Project: http://git-wip-us.apache.org/repos/asf/sentry/repo
Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/c4d3b5ea
Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/c4d3b5ea
Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/c4d3b5ea
Branch: refs/heads/sentry-ha-redesign
Commit: c4d3b5eae26f615cee02f7e77c642ff6dd7f0706
Parents: 79cbd17
Author: hahao <ha...@cloudera.com>
Authored: Fri Jan 20 11:31:26 2017 -0800
Committer: hahao <ha...@cloudera.com>
Committed: Fri Jan 20 11:31:26 2017 -0800
----------------------------------------------------------------------
.../sentry/service/thrift/HMSFollower.java | 57 ++++++++++----------
.../sentry/service/thrift/SentryService.java | 2 +-
2 files changed, 30 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sentry/blob/c4d3b5ea/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java
index f3cefd6..749c2ce 100644
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java
@@ -49,11 +49,11 @@ import java.util.List;
import static org.apache.sentry.binding.hive.conf.HiveAuthzConf.AuthzConfVars.AUTHZ_SYNC_CREATE_WITH_POLICY_STORE;
import static org.apache.sentry.binding.hive.conf.HiveAuthzConf.AuthzConfVars.AUTHZ_SYNC_DROP_WITH_POLICY_STORE;
-/*
-HMSFollower is the thread which follows the Hive MetaStore state changes from Sentry.
-It gets the full update and notification logs from HMS and applies it to
-update permissions stored in Sentry using SentryStore and also update the <obj,path> state
-stored for HDFS- Sentry sync.
+/**
+ * HMSFollower is the thread which follows the Hive MetaStore state changes from Sentry.
+ * It gets the full update and notification logs from HMS and applies it to
+ * update permissions stored in Sentry using SentryStore and also update the < obj,path > state
+ * stored for HDFS-Sentry sync.
*/
@SuppressWarnings("PMD")
public class HMSFollower implements Runnable {
@@ -63,19 +63,22 @@ public class HMSFollower implements Runnable {
private static boolean connectedToHMS = false;
private HiveMetaStoreClient client;
private SentryKerberosContext kerberosContext;
- private Configuration authzConf;
+ private final Configuration authzConf;
private boolean kerberos;
- private SentryStore sentryStore;
+ private final SentryStore sentryStore;
private String hiveInstance;
- final static int maxRetriesForLogin = 3;
- final static int maxRetriesForConnection = 3;
+ private static final int maxRetriesForLogin = 3;
+ private static final int maxRetriesForConnection = 3;
private volatile UpdateableAuthzPaths authzPaths;
private boolean needHiveSnapshot = true;
+ private final LeaderStatusMonitor leaderMonitor;
- HMSFollower(Configuration conf) throws Exception { //TODO: Handle any possible exceptions or throw specific exceptions
+ HMSFollower(Configuration conf, LeaderStatusMonitor leaderMonitor)
+ throws Exception {
LOGGER.info("HMSFollower is being initialized");
authzConf = conf;
+ this.leaderMonitor = leaderMonitor;
sentryStore = new SentryStore(authzConf);
//TODO: Initialize currentEventID from Sentry db
currentEventID = 0;
@@ -86,6 +89,7 @@ public class HMSFollower implements Runnable {
this.authzConf = conf;
this.sentryStore = sentryStore;
this.hiveInstance = hiveInstance;
+ this.leaderMonitor = null;
}
@VisibleForTesting
@@ -93,10 +97,10 @@ public class HMSFollower implements Runnable {
return connectedToHMS;
}
- /*
- Returns HMS Client if successful, returns null if HMS is not ready yet to take connections
- Throws @LoginException if Kerberos context creation failed using Sentry's kerberos credentials
- Throws @MetaException if there was a problem on creating an HMSClient
+ /**
+ * Returns HMS Client if successful, returns null if HMS is not ready yet to take connections
+ * Throws @LoginException if Kerberos context creation failed using Sentry's kerberos credentials
+ * Throws @MetaException if there was a problem on creating an HMSClient
*/
private HiveMetaStoreClient getMetaStoreClient(Configuration conf)
throws LoginException, MetaException {
@@ -179,6 +183,7 @@ public class HMSFollower implements Runnable {
return client;
}
+ @Override
public void run() {
if (client == null) {
try {
@@ -196,8 +201,13 @@ public class HMSFollower implements Runnable {
}
}
+ // Only the leader should listen to HMS updates
+ if ((leaderMonitor != null) && !leaderMonitor.isLeader()) {
+ return;
+ }
+
try {
- if (isNeedHiveSnapshot()) {
+ if (needHiveSnapshot) {
// TODO: expose time used for full update in the metrics
// To ensure point-in-time snapshot consistency, need to make sure
@@ -262,7 +272,8 @@ public class HMSFollower implements Runnable {
try {
updateInitializer = new FullUpdateInitializer(client, authzConf);
- HMSFollower.this.authzPaths = updateInitializer.createInitialUpdate();
+ // TODO - do we need to save returned authz path?
+ updateInitializer.createInitialUpdate();
// TODO: notify HDFS plugin
LOGGER.info("#### Hive full update initialization complete !!");
} finally {
@@ -276,24 +287,14 @@ public class HMSFollower implements Runnable {
}
}
- private boolean isNeedHiveSnapshot() {
- // An indicator that in request of a full hive update.
-
- // TODO: Will need to get Hive snapshot if the Notification ID
- // we are requesting has been rolled over in the NotificationLog
- // table of Hive
- return needHiveSnapshot;
- }
-
private boolean syncWithPolicyStore(HiveAuthzConf.AuthzConfVars syncConfVar) {
return "true"
.equalsIgnoreCase((authzConf.get(syncConfVar.getVar(), "true")));
}
- /*
- Throws SentryInvalidHMSEventException if Notification event contains insufficient information
+ /**
+ * Throws SentryInvalidHMSEventException if Notification event contains insufficient information
*/
-
void processNotificationEvents(List<NotificationEvent> events) throws
SentryInvalidHMSEventException, SentryInvalidInputException {
SentryJSONMessageDeserializer deserializer = new SentryJSONMessageDeserializer();
http://git-wip-us.apache.org/repos/asf/sentry/blob/c4d3b5ea/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java
index a89d3f6..be59c1e 100644
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java
@@ -158,7 +158,7 @@ public class SentryService implements Callable, SigUtils.SigListener {
if (notificationLogEnabled) {
try {
hmsFollowerExecutor = Executors.newScheduledThreadPool(1);
- hmsFollowerExecutor.scheduleAtFixedRate(new HMSFollower(conf),
+ hmsFollowerExecutor.scheduleAtFixedRate(new HMSFollower(conf, leaderMonitor),
60000, 500, TimeUnit.MILLISECONDS);
} catch (Exception e) {
//TODO: Handle