You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sentry.apache.org by as...@apache.org on 2015/03/04 20:54:49 UTC
incubator-sentry git commit: SENTRY-664: Fix to ensure path updates
are synced after namenode restart (Reviewed by Prasad)
Repository: incubator-sentry
Updated Branches:
refs/heads/master dd9edc666 -> feb8cbee6
SENTRY-664: Fix to ensure path updates are synced after namenode restart (Reviewed by Prasad)
Project: http://git-wip-us.apache.org/repos/asf/incubator-sentry/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-sentry/commit/feb8cbee
Tree: http://git-wip-us.apache.org/repos/asf/incubator-sentry/tree/feb8cbee
Diff: http://git-wip-us.apache.org/repos/asf/incubator-sentry/diff/feb8cbee
Branch: refs/heads/master
Commit: feb8cbee60e2d8d36667737d527c602de3e15d13
Parents: dd9edc6
Author: Arun Suresh <Arun Suresh>
Authored: Wed Mar 4 11:52:53 2015 -0800
Committer: Arun Suresh <Arun Suresh>
Committed: Wed Mar 4 11:52:53 2015 -0800
----------------------------------------------------------------------
.../sentry/hdfs/SentryAuthorizationInfo.java | 24 ++++++++++++++------
.../org/apache/sentry/hdfs/MetastorePlugin.java | 9 ++++----
.../tests/e2e/hdfs/TestHDFSIntegration.java | 12 ++++++++++
3 files changed, 33 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/feb8cbee/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryAuthorizationInfo.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryAuthorizationInfo.java b/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryAuthorizationInfo.java
index 2415890..f9a1f65 100644
--- a/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryAuthorizationInfo.java
+++ b/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryAuthorizationInfo.java
@@ -139,14 +139,14 @@ public class SentryAuthorizationInfo implements Runnable {
if ((newAuthzPaths != authzPaths)||(newAuthzPerms != authzPermissions)) {
lock.writeLock().lock();
try {
+ LOG.warn("FULL Updated paths seq Num [old="
+ + authzPaths.getLastUpdatedSeqNum() + "], [new="
+ + newAuthzPaths.getLastUpdatedSeqNum() + "]");
authzPaths = newAuthzPaths;
- if (LOG.isDebugEnabled()) {
- LOG.debug("FULL Updated paths seq Num [" + authzPaths.getLastUpdatedSeqNum() + "]");
- }
+ LOG.warn("FULL Updated perms seq Num [old="
+ + authzPermissions.getLastUpdatedSeqNum() + "], [new="
+ + newAuthzPerms.getLastUpdatedSeqNum() + "]");
authzPermissions = newAuthzPerms;
- if (LOG.isDebugEnabled()) {
- LOG.debug("FULL Updated perms seq Num [" + authzPermissions.getLastUpdatedSeqNum() + "]");
- }
} finally {
lock.writeLock().unlock();
}
@@ -162,12 +162,22 @@ public class SentryAuthorizationInfo implements Runnable {
// one in the List.. all the remaining will be partial updates
if (updates.size() > 0) {
if (updates.get(0).hasFullImage()) {
+ LOG.warn("Process Update : FULL IMAGE "
+ + "[" + updateable.getClass() + "]"
+ + "[" + updates.get(0).getSeqNum() + "]");
updateable = (V)updateable.updateFull(updates.remove(0));
}
// Any more elements ?
if (!updates.isEmpty()) {
+ LOG.warn("Process Update : More updates.. "
+ + "[" + updateable.getClass() + "]"
+ + "[" + updateable.getLastUpdatedSeqNum() + "]"
+ + "[" + updates.size() + "]");
updateable.updatePartial(updates, lock);
}
+ LOG.warn("Process Update : Finished updates.. "
+ + "[" + updateable.getClass() + "]"
+ + "[" + updateable.getLastUpdatedSeqNum() + "]");
}
return updateable;
}
@@ -198,7 +208,7 @@ public class SentryAuthorizationInfo implements Runnable {
}
public void start() {
- if (authzPaths != null) {
+ if ((authzPaths != null)||(authzPermissions != null)) {
boolean success = false;
try {
success = update();
http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/feb8cbee/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/MetastorePlugin.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/MetastorePlugin.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/MetastorePlugin.java
index f4964d6..5277eef 100644
--- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/MetastorePlugin.java
+++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/MetastorePlugin.java
@@ -88,12 +88,11 @@ public class MetastorePlugin extends SentryMetastoreListenerPlugin {
private UpdateableAuthzPaths authzPaths;
private Lock notificiationLock;
- //Initialized to some value > 1 so that the first update notification
- // will trigger a full Image fetch
- private final AtomicLong seqNum = new AtomicLong(5);
+ // Initialized to some value > 1.
+ private static final AtomicLong seqNum = new AtomicLong(5);
- // For some reason, HMS sometimes restarts the plugin
- private static volatile long lastSentSeqNum = -1;
+ // Has to match the value of seqNum
+ private static volatile long lastSentSeqNum = seqNum.get();
private volatile boolean syncSent = false;
private final ExecutorService threadPool;
http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/feb8cbee/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegration.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegration.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegration.java
index 4cb7275..955c68a 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegration.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegration.java
@@ -764,6 +764,18 @@ public class TestHDFSIntegration {
verifyOnPath("/tmp/external/tables/ext2_after/i=2/stuff.txt", FsAction.ALL, "hbase", true);
// END : Verify external table set location..
+ // Restart HDFS to verify if things are fine after re-start..
+
+ // TODO : this is currently commented out since miniDFS.restartNameNode() does
+ // not work corectly on the version of hadoop sentry depends on
+ // This has been verified to work on a real cluster.
+ // Once miniDFS is fixed, this should be uncommented..
+ // miniDFS.shutdown();
+ // miniDFS.restartNameNode(true);
+ // miniDFS.waitActive();
+ // verifyOnPath("/tmp/external/tables/ext2_after", FsAction.ALL, "hbase", true);
+ // verifyOnAllSubDirs("/user/hive/warehouse/p2", FsAction.READ_EXECUTE, "hbase", true);
+
stmt.close();
conn.close();
}