You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sentry.apache.org by vs...@apache.org on 2017/10/03 19:48:32 UTC
sentry git commit: SENTRY-1966: Improve logging of HMS sync data
(paths and permissions) flowing from Sentry to NameNode (Vadim Spector,
reviewed by Arjun Mishra)
Repository: sentry
Updated Branches:
refs/heads/master 186e75543 -> 19efc0e44
SENTRY-1966: Improve logging of HMS sync data (paths and permissions) flowing from Sentry to NameNode (Vadim Spector, reviewed by Arjun Mishra)
Project: http://git-wip-us.apache.org/repos/asf/sentry/repo
Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/19efc0e4
Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/19efc0e4
Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/19efc0e4
Branch: refs/heads/master
Commit: 19efc0e44ad71e796c8fc8cac1cbd30b988f8ebb
Parents: 186e755
Author: Vadim Spector <vs...@cloudera.com>
Authored: Tue Oct 3 12:29:05 2017 -0700
Committer: Vadim Spector <vs...@cloudera.com>
Committed: Tue Oct 3 12:45:31 2017 -0700
----------------------------------------------------------------------
.../java/org/apache/sentry/hdfs/HMSPaths.java | 142 +++++++++++++++----
.../org/apache/sentry/hdfs/PathsUpdate.java | 6 +
.../apache/sentry/hdfs/PermissionsUpdate.java | 6 +
.../apache/sentry/hdfs/SentryAuthzUpdate.java | 27 ++++
.../sentry/hdfs/UpdateableAuthzPaths.java | 9 ++
.../sentry/hdfs/SentryAuthorizationInfo.java | 59 ++++++--
.../hdfs/SentryINodeAttributesProvider.java | 2 +-
.../apache/sentry/hdfs/SentryPermissions.java | 25 ++++
.../sentry/hdfs/UpdateableAuthzPermissions.java | 9 ++
.../SentryHDFSServiceClientDefaultImpl.java | 23 +++
10 files changed, 266 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sentry/blob/19efc0e4/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/HMSPaths.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/HMSPaths.java b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/HMSPaths.java
index 5e975cb..3919d60 100644
--- a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/HMSPaths.java
+++ b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/HMSPaths.java
@@ -70,6 +70,22 @@ public class HMSPaths implements AuthzPaths {
return pathsElements;
}
+ // used for more compact logging
+ static List<String> assemblePaths(List<List<String>> pathElements) {
+ if (pathElements == null) {
+ return Collections.emptyList();
+ }
+ List<String> paths = new ArrayList<>(pathElements.size());
+ for (List<String> path : pathElements) {
+ StringBuffer sb = new StringBuffer();
+ for (String elem : path) {
+ sb.append(Path.SEPARATOR_CHAR).append(elem);
+ }
+ paths.add(sb.toString());
+ }
+ return paths;
+ }
+
@VisibleForTesting
enum EntryType {
DIR(true),
@@ -252,9 +268,10 @@ public class HMSPaths implements AuthzPaths {
parent = null;
}
+ @Override
public String toString() {
- return String.format("Entry[fullPath: %s, type: %s, authObject: %s]",
- getFullPath(), type, authzObjsToString());
+ return String.format("Entry[%s:%s -> authObj: %s]",
+ type, getFullPath(), authzObjsToString());
}
private String authzObjsToString() {
@@ -408,8 +425,8 @@ public class HMSPaths implements AuthzPaths {
Entry prefix = findPrefixEntry(pathElements);
if (prefix != null) {
throw new IllegalArgumentException(String.format(
- "Cannot add prefix '%s' under an existing prefix '%s'",
- toPath(pathElements), prefix.getFullPath()));
+ "%s: createPrefix(%s): cannot add prefix under an existing prefix '%s'",
+ this, pathElements, prefix.getFullPath()));
}
return createChild(pathElements, EntryType.PREFIX, null);
}
@@ -421,7 +438,10 @@ public class HMSPaths implements AuthzPaths {
// we only create the entry if is under a prefix, else we ignore it
entry = createChild(pathElements, EntryType.AUTHZ_OBJECT, authzObj);
} else {
- LOG.debug("Skipping to create authzObjPath as it is outside of prefix. authObj={} pathElements={}", authzObj, pathElements);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(String.format("%s: createAuthzObjPath(%s, %s): outside of prefix, skipping",
+ this, authzObj, pathElements));
+ }
}
return entry;
}
@@ -478,7 +498,7 @@ public class HMSPaths implements AuthzPaths {
if (newParent.getChild(pathElem) != null) {
LOG.warn(String.format(
"Attempt to move %s to %s: entry with the same name %s already exists",
- this.getFullPath(), newParent.getFullPath(), pathElem));
+ this, newParent, pathElem));
return;
}
deleteFromParent();
@@ -642,6 +662,7 @@ public class HMSPaths implements AuthzPaths {
private Map<String, Set<Entry>> authzObjToEntries;
public HMSPaths() {
+ LOG.info(toString() + " (default) Initialized");
}
public HMSPaths(String[] pathPrefixes) {
@@ -663,6 +684,7 @@ public class HMSPaths implements AuthzPaths {
}
authzObjToEntries = new TreeMap<String, Set<Entry>>(String.CASE_INSENSITIVE_ORDER);
+ LOG.info(toString() + " Initialized");
}
void _addAuthzObject(String authzObj, List<String> authzObjPaths) {
@@ -670,6 +692,10 @@ public class HMSPaths implements AuthzPaths {
}
void addAuthzObject(String authzObj, List<List<String>> authzObjPathElements) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(String.format("%s addAuthzObject(%s, %s)",
+ this, authzObj, assemblePaths(authzObjPathElements)));
+ }
Set<Entry> previousEntries = authzObjToEntries.get(authzObj);
Set<Entry> newEntries = new HashSet<Entry>(authzObjPathElements.size());
for (List<String> pathElements : authzObjPathElements) {
@@ -677,7 +703,9 @@ public class HMSPaths implements AuthzPaths {
if (e != null) {
newEntries.add(e);
} else {
- LOG.warn("Ignoring path, no prefix");
+ LOG.warn(String.format("%s addAuthzObject(%s, %s):" +
+ " Ignoring path %s, no prefix",
+ this, authzObj, assemblePaths(authzObjPathElements), pathElements));
}
}
authzObjToEntries.put(authzObj, newEntries);
@@ -693,6 +721,10 @@ public class HMSPaths implements AuthzPaths {
void addPathsToAuthzObject(String authzObj,
List<List<String>> authzObjPathElements, boolean createNew) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(String.format("%s addPathsToAuthzObject(%s, %s, %b)",
+ this, authzObj, assemblePaths(authzObjPathElements), createNew));
+ }
Set<Entry> entries = authzObjToEntries.get(authzObj);
if (entries != null) {
Set<Entry> newEntries = new HashSet<Entry>(authzObjPathElements.size());
@@ -701,7 +733,11 @@ public class HMSPaths implements AuthzPaths {
if (e != null) {
newEntries.add(e);
} else {
- LOG.debug("Cannot create authz obj path for {} because it is outside of prefix", authzObj);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(String.format("%s addPathsToAuthzObject(%s, %s, %b):" +
+ " Cannot create authz obj for path %s because it is outside of prefix",
+ this, authzObj, assemblePaths(authzObjPathElements), createNew, pathElements));
+ }
}
}
entries.addAll(newEntries);
@@ -709,8 +745,9 @@ public class HMSPaths implements AuthzPaths {
if (createNew) {
addAuthzObject(authzObj, authzObjPathElements);
} else {
- LOG.warn("Path was not added to AuthzObject, could not find key in authzObjToPath. authzObj = " + authzObj +
- " authzObjPathElements=" + authzObjPathElements);
+ LOG.warn(String.format("%s addPathsToAuthzObject(%s, %s, %b):" +
+ " Path was not added to AuthzObject, could not find key in authzObjToPath",
+ this, authzObj, assemblePaths(authzObjPathElements), createNew));
}
}
}
@@ -730,6 +767,10 @@ public class HMSPaths implements AuthzPaths {
*/
void deletePathsFromAuthzObject(String authzObj,
List<List<String>> authzObjPathElements) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(String.format("%s deletePathsFromAuthzObject(%s, %s)",
+ this, authzObj, assemblePaths(authzObjPathElements)));
+ }
Set<Entry> entries = authzObjToEntries.get(authzObj);
if (entries != null) {
Set<Entry> toDelEntries = new HashSet<Entry>(authzObjPathElements.size());
@@ -740,17 +781,24 @@ public class HMSPaths implements AuthzPaths {
entry.deleteAuthzObject(authzObj);
toDelEntries.add(entry);
} else {
- LOG.info("Path was not deleted from AuthzObject, path not registered. This is possible for implicit partition locations. authzObj = " + authzObj + " authzObjPathElements=" + authzObjPathElements);
+ LOG.warn(String.format("%s deletePathsFromAuthzObject(%s, %s):" +
+ " Path %s was not deleted from AuthzObject, path not registered." +
+ " This is possible for implicit partition locations",
+ this, authzObj, assemblePaths(authzObjPathElements), pathElements));
}
}
entries.removeAll(toDelEntries);
} else {
- LOG.info("Path was not deleted from AuthzObject, could not find key in authzObjToPath. authzObj = " + authzObj +
- " authzObjPathElements=" + authzObjPathElements);
+ LOG.warn(String.format("%s deletePathsFromAuthzObject(%s, %s):" +
+ " Path was not deleted from AuthzObject, could not find key in authzObjToPath",
+ this, authzObj, assemblePaths(authzObjPathElements)));
}
}
void deleteAuthzObject(String authzObj) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(String.format("%s deleteAuthzObject(%s)", this, authzObj));
+ }
Set<Entry> entries = authzObjToEntries.remove(authzObj);
if (entries != null) {
for (Entry entry : entries) {
@@ -784,12 +832,21 @@ public class HMSPaths implements AuthzPaths {
* @return Returns a set of authzObjects authzObject associated with this path.
*/
public Set<String> findAuthzObject(String[] pathElements, boolean isPartialOk) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(String.format("%s findAuthzObject(%s, %b)",
+ this, Arrays.toString(pathElements), isPartialOk));
+ }
// Handle '/'
if (pathElements == null || pathElements.length == 0) {
return null;
}
Entry entry = root.find(pathElements, isPartialOk);
- return (entry != null) ? entry.getAuthzObjs() : null;
+ Set<String> authzObjSet = (entry != null) ? entry.getAuthzObjs() : null;
+ if ((authzObjSet == null || authzObjSet.isEmpty()) && LOG.isDebugEnabled()) {
+ LOG.debug(String.format("%s findAuthzObject(%s, %b) - no authzObject found",
+ this, Arrays.toString(pathElements), isPartialOk));
+ }
+ return authzObjSet;
}
/*
@@ -801,14 +858,16 @@ public class HMSPaths implements AuthzPaths {
*/
void renameAuthzObject(String oldName, List<List<String>> oldPathElems,
String newName, List<List<String>> newPathElems) {
-
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(String.format("%s renameAuthzObject({%s, %s} -> {%s, %s})",
+ this, oldName, assemblePaths(oldPathElems), newName, assemblePaths(newPathElems)));
+ }
if (oldPathElems == null || oldPathElems.isEmpty() ||
newPathElems == null || newPathElems.isEmpty() ||
newName == null || newName.equals(oldName)) {
- LOG.warn(String.format(
- "Unexpected state in renameAuthzObject, inputs invalid: " +
- "oldName=%s newName=%s oldPath=%s newPath=%s",
- oldName, newName, oldPathElems, newPathElems));
+ LOG.warn(String.format("%s renameAuthzObject({%s, %s} -> {%s, %s})" +
+ ": invalid inputs, skipping",
+ this, oldName, assemblePaths(oldPathElems), newName, assemblePaths(newPathElems)));
return;
}
@@ -825,9 +884,9 @@ public class HMSPaths implements AuthzPaths {
// Re-write authObj from oldName to newName.
Set<Entry> entries = authzObjToEntries.get(oldName);
if (entries == null) {
- LOG.warn("Unexpected state in renameAuthzObject, cannot find oldName in authzObjToPath: " +
- "oldName=" + oldName + " newName=" + newName +
- " oldPath=" + oldPathElems + " newPath=" + newPathElems);
+ LOG.warn(String.format("%s renameAuthzObject({%s, %s} -> {%s, %s}):" +
+ " cannot find oldName %s in authzObjToPath",
+ this, oldName, assemblePaths(oldPathElems), newName, assemblePaths(newPathElems), oldName));
} else {
authzObjToEntries.put(newName, entries);
for (Entry e : entries) {
@@ -836,10 +895,10 @@ public class HMSPaths implements AuthzPaths {
if (e.getAuthzObjs().contains(oldName)) {
e.removeAuthzObj(oldName);
} else {
- LOG.warn("Unexpected state in renameAuthzObject, authzObjToPath has an " +
- "entry <oldName,entries> where one of the entry does not have oldName : " +
- "oldName=" + oldName + " newName=" + newName +
- " oldPath=" + oldPathElems + " newPath=" + newPathElems);
+ LOG.warn(String.format("%s renameAuthzObject({%s, %s} -> {%s, %s}):" +
+ " Unexpected state: authzObjToPath has an " +
+ "entry %s where one of the authz objects does not have oldName",
+ this, oldName, assemblePaths(oldPathElems), newName, assemblePaths(newPathElems), e));
}
}
}
@@ -870,9 +929,40 @@ public class HMSPaths implements AuthzPaths {
authzObjToEntries = mapping;
}
+ /**
+ * For logging: collect all path entries into a list.
+ *
+ * Each Entry has informative toString() implementation,
+ * so we can print the returned value directly.
+ *
+ * Non-recursive traversal.
+ */
+ public Collection<Entry> getAllEntries() {
+ Collection<Entry> entries = new ArrayList<>();
+ Stack<Entry> stack = new Stack<>();
+ stack.push(root);
+ while (!stack.isEmpty()) {
+ Entry entry = stack.pop();
+ entries.add(entry);
+ for (Entry child : entry.childrenValues()) { // handles entry.children == null
+ stack.push(child);
+ }
+ }
+ return entries;
+ }
+
@Override
public HMSPathsDumper getPathsDump() {
return new HMSPathsDumper(this);
}
+ @Override
+ public String toString() {
+ return String.format("%s:%s", getClass().getSimpleName(), Arrays.toString(prefixes));
+ }
+
+ public String dumpContent() {
+ return toString() + ": " + getAllEntries();
+ }
+
}
http://git-wip-us.apache.org/repos/asf/sentry/blob/19efc0e4/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java
index 3996747..5ff2dc2 100644
--- a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java
+++ b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java
@@ -216,4 +216,10 @@ public class PathsUpdate implements Updateable.Update {
return tPathsUpdate.equals(other.tPathsUpdate);
}
+ @Override
+ public String toString() {
+ // TPathsUpdate implements toString() perfectly; null tPathsUpdate is ok
+ return getClass().getSimpleName() + "(" + tPathsUpdate + ")";
+ }
+
}
http://git-wip-us.apache.org/repos/asf/sentry/blob/19efc0e4/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PermissionsUpdate.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PermissionsUpdate.java b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PermissionsUpdate.java
index 392f9a6..0272396 100644
--- a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PermissionsUpdate.java
+++ b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PermissionsUpdate.java
@@ -155,4 +155,10 @@ public class PermissionsUpdate implements Updateable.Update {
return tPermUpdate.equals(other.tPermUpdate);
}
+ @Override
+ public String toString() {
+ // TPermissionsUpdate implements toString() perfectly; null tPermUpdate is ok
+ return getClass().getSimpleName() + "(" + tPermUpdate + ")";
+ }
+
}
http://git-wip-us.apache.org/repos/asf/sentry/blob/19efc0e4/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/SentryAuthzUpdate.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/SentryAuthzUpdate.java b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/SentryAuthzUpdate.java
index 4cf439b..db11c1e 100644
--- a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/SentryAuthzUpdate.java
+++ b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/SentryAuthzUpdate.java
@@ -38,4 +38,31 @@ public class SentryAuthzUpdate {
return pathUpdates;
}
+ public String dumpContent() {
+ StringBuffer sb = new StringBuffer(getClass().getSimpleName());
+ if (permUpdates != null && !permUpdates.isEmpty()) {
+ sb.append(", perms[").append(permUpdates.size()).append(']').append(permUpdates);
+ }
+ if (pathUpdates != null && !pathUpdates.isEmpty()) {
+ sb.append(", paths[").append(pathUpdates.size()).append(']').append(pathUpdates);
+ }
+ return sb.toString();
+ }
+
+ public boolean isEmpty() {
+ return (permUpdates == null || permUpdates.isEmpty()) &&
+ (pathUpdates == null || pathUpdates.isEmpty());
+ }
+
+ @Override
+ public String toString() {
+ StringBuffer sb = new StringBuffer(getClass().getSimpleName());
+ if (permUpdates != null && !permUpdates.isEmpty()) {
+ sb.append(", perms[").append(permUpdates.size()).append(']');
+ }
+ if (pathUpdates != null && !pathUpdates.isEmpty()) {
+ sb.append(", paths[").append(pathUpdates.size()).append(']');
+ }
+ return sb.toString();
+ }
}
http://git-wip-us.apache.org/repos/asf/sentry/blob/19efc0e4/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/UpdateableAuthzPaths.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/UpdateableAuthzPaths.java b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/UpdateableAuthzPaths.java
index cd97939..c967f4e 100644
--- a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/UpdateableAuthzPaths.java
+++ b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/UpdateableAuthzPaths.java
@@ -191,4 +191,13 @@ public class UpdateableAuthzPaths implements AuthzPaths, Updateable<PathsUpdate>
public String getUpdateableTypeName() {
return UPDATABLE_TYPE_NAME;
}
+
+ @Override
+ public String toString() {
+ return String.format("%s(%s, %s, %s)", getClass().getSimpleName(), seqNum, imgNum, paths);
+ }
+
+ public String dumpContent() {
+ return String.format("%s(%s, %s) ", getClass().getSimpleName(), seqNum, imgNum) + paths.dumpContent();
+ }
}
http://git-wip-us.apache.org/repos/asf/sentry/blob/19efc0e4/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryAuthorizationInfo.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryAuthorizationInfo.java b/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryAuthorizationInfo.java
index 2724a55..61b0d10 100644
--- a/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryAuthorizationInfo.java
+++ b/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryAuthorizationInfo.java
@@ -87,15 +87,13 @@ public class SentryAuthorizationInfo implements Runnable {
SentryAuthorizationConstants.CACHE_REFRESH_RETRY_WAIT_KEY,
SentryAuthorizationConstants.CACHE_REFRESH_RETRY_WAIT_DEFAULT);
- if (LOG.isDebugEnabled()) {
- LOG.debug(
- "Sentry authorization will enforced in the following HDFS locations: [{}]",
+ LOG.info("Sentry authorization will enforced in the following HDFS locations: [{}]",
StringUtils.arrayToString(newPathPrefixes));
- }
+
setPrefixPaths(newPathPrefixes);
- LOG.debug("Refresh interval [{}]ms, retry wait [{}]",
+ LOG.info("Refresh interval [{}]ms, retry wait [{}]",
refreshIntervalMillisec, retryWaitMillisec);
- LOG.debug("stale threshold [{}]ms", staleThresholdMillisec);
+ LOG.info("stale threshold [{}]ms", staleThresholdMillisec);
authzPaths = new UpdateableAuthzPaths(newPathPrefixes);
authzPermissions = new UpdateableAuthzPermissions();
@@ -134,6 +132,10 @@ public class SentryAuthorizationInfo implements Runnable {
SentryAuthzUpdate updates = updater.getUpdates();
// Updates can be null if Sentry Service is un-reachable
if (updates != null) {
+ if (updates.isEmpty()) {
+ return true; // no updates is a norm, it's still success
+ }
+
UpdateableAuthzPaths newAuthzPaths = processUpdates(
updates.getPathUpdates(), authzPaths);
UpdateableAuthzPermissions newAuthzPerms = processUpdates(
@@ -145,16 +147,41 @@ public class SentryAuthorizationInfo implements Runnable {
if (newAuthzPaths != authzPaths || newAuthzPerms != authzPermissions) {
lock.writeLock().lock();
try {
- LOG.debug(String.format("FULL Updated paths seq Num [old=%d], [new=%d] img Num [old=%d], [new=%d]",
- authzPaths.getLastUpdatedSeqNum(), newAuthzPaths.getLastUpdatedSeqNum(),
- authzPaths.getLastUpdatedImgNum(), newAuthzPaths.getLastUpdatedImgNum()));
- authzPaths = newAuthzPaths;
- LOG.debug(String.format("FULL Updated perms seq Num [old=%d], [new=%d]",
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(updates.dumpContent());
+ }
+ if (newAuthzPaths != authzPaths) {
+ LOG.info(String.format("FULL Updated paths seq Num [old=%d], [new=%d]",
+ authzPaths.getLastUpdatedSeqNum(), newAuthzPaths.getLastUpdatedSeqNum()));
+ authzPaths = newAuthzPaths;
+ if (LOG.isTraceEnabled()) {
+ LOG.trace(authzPaths.dumpContent());
+ }
+ }
+ if (newAuthzPerms != authzPermissions) {
+ LOG.info(String.format("FULL Updated perms seq Num [old=%d], [new=%d]",
authzPermissions.getLastUpdatedSeqNum(), newAuthzPerms.getLastUpdatedSeqNum()));
- authzPermissions = newAuthzPerms;
+ authzPermissions = newAuthzPerms;
+ if (LOG.isTraceEnabled()) {
+ LOG.trace(authzPermissions.dumpContent());
+ }
+ }
} finally {
lock.writeLock().unlock();
}
+ } else {
+ if (LOG.isDebugEnabled()) {
+ lock.writeLock().lock();
+ try {
+ LOG.debug(updates.dumpContent());
+ if (LOG.isTraceEnabled()) {
+ LOG.trace(newAuthzPaths.dumpContent());
+ LOG.trace(newAuthzPerms.dumpContent());
+ }
+ } finally {
+ lock.writeLock().unlock();
+ }
+ }
}
return true;
}
@@ -169,7 +196,7 @@ public class SentryAuthorizationInfo implements Runnable {
if (!updates.isEmpty()) {
if (updates.get(0).hasFullImage()) {
LOG.debug(String.format("Process Update : FULL IMAGE [%s][%d][%d]",
- newUpdateable.getClass().getName(),
+ newUpdateable.getClass().getSimpleName(),
updates.get(0).getSeqNum(),
updates.get(0).getImgNum()));
newUpdateable = (V)newUpdateable.updateFull(updates.remove(0));
@@ -177,20 +204,21 @@ public class SentryAuthorizationInfo implements Runnable {
// Any more elements ?
if (!updates.isEmpty()) {
LOG.debug(String.format("Process Update : More updates.. [%s][%d][%d][%d]",
- newUpdateable.getClass().getName(),
+ newUpdateable.getClass().getSimpleName(),
newUpdateable.getLastUpdatedSeqNum(),
newUpdateable.getLastUpdatedImgNum(),
updates.size()));
newUpdateable.updatePartial(updates, lock);
}
LOG.debug(String.format("Process Update : Finished updates.. [%s][%d][%d]",
- newUpdateable.getClass().getName(),
+ newUpdateable.getClass().getSimpleName(),
newUpdateable.getLastUpdatedSeqNum(),
newUpdateable.getLastUpdatedImgNum()));
}
return newUpdateable;
}
+ @Override
public void run() {
boolean success = false;
try {
@@ -241,6 +269,7 @@ public class SentryAuthorizationInfo implements Runnable {
public void stop() {
if (authzPaths != null) {
+ LOG.info(getClass().getSimpleName() + ": Stopping");
executor.shutdownNow();
}
}
http://git-wip-us.apache.org/repos/asf/sentry/blob/19efc0e4/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryINodeAttributesProvider.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryINodeAttributesProvider.java b/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryINodeAttributesProvider.java
index 23b831d..cf96df4 100644
--- a/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryINodeAttributesProvider.java
+++ b/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryINodeAttributesProvider.java
@@ -323,7 +323,7 @@ public class SentryINodeAttributesProvider extends INodeAttributeProvider
@Override
public void stop() {
- LOG.debug("Stopping");
+ LOG.debug(getClass().getSimpleName() + ": Stopping");
authzInfo.stop();
}
http://git-wip-us.apache.org/repos/asf/sentry/blob/19efc0e4/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryPermissions.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryPermissions.java b/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryPermissions.java
index c01ff68..3b3a82e 100644
--- a/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryPermissions.java
+++ b/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryPermissions.java
@@ -49,6 +49,10 @@ public class SentryPermissions implements AuthzPermissions {
public String getAuthzObj() {
return authzObj;
}
+ @Override
+ public String toString() {
+ return "PrivilegeInfo(" + authzObj + " --> " + roleToPermission + ")";
+ }
}
public static class RoleInfo {
@@ -71,6 +75,10 @@ public class SentryPermissions implements AuthzPermissions {
public Set<String> getAllGroups() {
return groups;
}
+ @Override
+ public String toString() {
+ return "RoleInfo(" + role + " --> " + groups + ")";
+ }
}
// Comparison of authorizable object should be case insensitive.
@@ -203,4 +211,21 @@ public class SentryPermissions implements AuthzPermissions {
public void addRoleInfo(RoleInfo roleInfo) {
roles.put(roleInfo.role, roleInfo);
}
+
+ public String dumpContent() {
+ return new StringBuffer(getClass().getSimpleName())
+ .append(": Privileges: ").append(privileges)
+ .append(", Roles: ").append(roles)
+ .append(", AuthzObjChildren: ").append(authzObjChildren)
+ .toString();
+ }
+
+ @Override
+ public String toString() {
+ return new StringBuffer(getClass().getSimpleName())
+ .append(": Privileges: ").append(privileges.size())
+ .append(", Roles: ").append(roles.size())
+ .append(", AuthzObjChildren: ").append(authzObjChildren.size())
+ .toString();
+ }
}
http://git-wip-us.apache.org/repos/asf/sentry/blob/19efc0e4/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/UpdateableAuthzPermissions.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/UpdateableAuthzPermissions.java b/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/UpdateableAuthzPermissions.java
index bf13ce7..1505513 100644
--- a/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/UpdateableAuthzPermissions.java
+++ b/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/UpdateableAuthzPermissions.java
@@ -250,4 +250,13 @@ public class UpdateableAuthzPermissions implements AuthzPermissions, Updateable<
return UPDATABLE_TYPE_NAME;
}
+ @Override
+ public String toString() {
+ return String.format("%s(%s, %s)", getClass().getSimpleName(), seqNum, perms);
+ }
+
+ public String dumpContent() {
+ return String.format("%s(%s) ", getClass().getSimpleName(), seqNum) + perms.dumpContent();
+ }
+
}
http://git-wip-us.apache.org/repos/asf/sentry/blob/19efc0e4/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHDFSServiceClientDefaultImpl.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHDFSServiceClientDefaultImpl.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHDFSServiceClientDefaultImpl.java
index 30d8adf..9115697 100644
--- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHDFSServiceClientDefaultImpl.java
+++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHDFSServiceClientDefaultImpl.java
@@ -37,6 +37,9 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import static org.apache.sentry.hdfs.service.thrift.sentry_hdfs_serviceConstants.UNUSED_PATH_UPDATE_IMG_NUM;
/**
@@ -46,6 +49,9 @@ import static org.apache.sentry.hdfs.service.thrift.sentry_hdfs_serviceConstants
*/
public class SentryHDFSServiceClientDefaultImpl
implements SentryHDFSServiceClient, SentryConnection {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(SentryHDFSServiceClientDefaultImpl.class);
+
private final boolean useCompactTransport;
private Client client;
private final SentryTransportPool transportPool;
@@ -97,6 +103,7 @@ public class SentryHDFSServiceClientDefaultImpl
try {
TAuthzUpdateRequest updateRequest = new TAuthzUpdateRequest(permSeqNum, pathSeqNum, pathImgNum);
TAuthzUpdateResponse sentryUpdates = client.get_authz_updates(updateRequest);
+
List<PathsUpdate> pathsUpdates = Collections.emptyList();
if (sentryUpdates.getAuthzPathUpdate() != null) {
pathsUpdates = new ArrayList<>(sentryUpdates.getAuthzPathUpdate().size());
@@ -113,6 +120,19 @@ public class SentryHDFSServiceClientDefaultImpl
}
}
+ if (LOGGER.isDebugEnabled() && !(permsUpdates.isEmpty() && pathsUpdates.isEmpty()) ) {
+ LOGGER.debug("getAllUpdatesFrom({},{},{}): permsUpdates[{}], pathsUpdates[{}]",
+ new Object[] { permSeqNum, pathSeqNum, pathImgNum, permsUpdates.size(), pathsUpdates.size() });
+ if (LOGGER.isTraceEnabled()) {
+ if (!permsUpdates.isEmpty()) {
+ LOGGER.trace("permsUpdates{}", permsUpdates);
+ }
+ if (!pathsUpdates.isEmpty()) {
+ LOGGER.trace("pathsUpdates{}", pathsUpdates);
+ }
+ }
+ }
+
return new SentryAuthzUpdate(permsUpdates, pathsUpdates);
} catch (Exception e) {
throw new SentryHdfsServiceException("Thrift Exception occurred !!", e);
@@ -136,6 +156,9 @@ public class SentryHDFSServiceClientDefaultImpl
public void invalidate() {
if (transport != null) {
transportPool.invalidateTransport(transport);
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("invalidate: " + transport);
+ }
transport = null;
}
}