You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sentry.apache.org by va...@apache.org on 2017/09/13 23:35:37 UTC
sentry git commit: SENTRY-1938: Sentry logs to provide more relevant
information (Arjun Mishra, Reviewed by: Vamsee Yarlagadda, Alexander Kolbasov)
Repository: sentry
Updated Branches:
refs/heads/master 213a37092 -> d4172ba9f
SENTRY-1938: Sentry logs to provide more relevant information (Arjun
Mishra, Reviewed by: Vamsee Yarlagadda, Alexander Kolbasov)
Project: http://git-wip-us.apache.org/repos/asf/sentry/repo
Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/d4172ba9
Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/d4172ba9
Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/d4172ba9
Branch: refs/heads/master
Commit: d4172ba9f957610637978da5dfa4032464d4aafc
Parents: 213a370
Author: Vamsee Yarlagadda <va...@cloudera.com>
Authored: Wed Sep 13 14:40:20 2017 -0700
Committer: Vamsee Yarlagadda <va...@cloudera.com>
Committed: Wed Sep 13 16:33:49 2017 -0700
----------------------------------------------------------------------
.../apache/sentry/hdfs/DBUpdateForwarder.java | 34 ++++++++++++++------
.../org/apache/sentry/hdfs/SentryPlugin.java | 12 ++++---
.../service/thrift/NotificationProcessor.java | 2 +-
3 files changed, 32 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sentry/blob/d4172ba9/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/DBUpdateForwarder.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/DBUpdateForwarder.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/DBUpdateForwarder.java
index 8a34d56..eae7861 100644
--- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/DBUpdateForwarder.java
+++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/DBUpdateForwarder.java
@@ -43,10 +43,14 @@ class DBUpdateForwarder<K extends Updateable.Update> {
private final DeltaRetriever<K> deltaRetriever;
private static final Logger LOGGER = LoggerFactory.getLogger(DBUpdateForwarder.class);
+ //For logging purposes
+ private String retrieverType;
+
DBUpdateForwarder(final ImageRetriever<K> imageRetriever,
- final DeltaRetriever<K> deltaRetriever) {
+ final DeltaRetriever<K> deltaRetriever) {
this.imageRetriever = imageRetriever;
this.deltaRetriever = deltaRetriever;
+ this.retrieverType = imageRetriever.getClass().getName();
}
/**
@@ -71,12 +75,12 @@ class DBUpdateForwarder<K extends Updateable.Update> {
* e.g. {@link PathsUpdate} or {@link PermissionsUpdate}
*/
List<K> getAllUpdatesFrom(long seqNum, long imgNum) throws Exception {
- LOGGER.debug("GetAllUpdatesFrom sequence number {}, image number {}", seqNum, imgNum);
+ LOGGER.debug(String.format("(%s) GetAllUpdatesFrom sequence number %d, image number %d", retrieverType, seqNum, imgNum));
// An imgNum >= 0 are valid values for image identifiers (0 means a full update is requested)
if (imgNum >= IMAGE_NUMBER_UPDATE_UNINITIALIZED) {
long curImgNum = imageRetriever.getLatestImageID();
- LOGGER.debug("Current image number is {}", curImgNum);
+ LOGGER.debug("({}) Current image number for is {}", retrieverType, curImgNum);
if (curImgNum == IMAGE_NUMBER_UPDATE_UNINITIALIZED) {
// Sentry has not fetched a full HMS snapshot yet.
@@ -85,8 +89,13 @@ class DBUpdateForwarder<K extends Updateable.Update> {
if (curImgNum > imgNum) {
// In case a new HMS snapshot has been processed, then return a full paths image.
- LOGGER.info("A newer full update is found with image number: {}", curImgNum);
- return retrieveFullImage();
+ List<K>fullImage = retrieveFullImage();
+ //Only log if we have received full image
+ if( !fullImage.isEmpty()) {
+ LOGGER.info("({}) A newer full update with image number {} was found and being sent to HDFS",
+ retrieverType, curImgNum);
+ }
+ return fullImage;
}
}
@@ -95,7 +104,7 @@ class DBUpdateForwarder<K extends Updateable.Update> {
*/
long curSeqNum = deltaRetriever.getLatestDeltaID();
- LOGGER.debug("Current sequence number is {}", curSeqNum);
+ LOGGER.debug("({}) Current sequence number is {}", retrieverType, curSeqNum);
if (seqNum > curSeqNum) {
// No new notifications were processed.
@@ -107,20 +116,25 @@ class DBUpdateForwarder<K extends Updateable.Update> {
if (seqNum > SEQUENCE_NUMBER_FULL_UPDATE_REQUEST && deltaRetriever.isDeltaAvailable(seqNum)) {
List<K> deltas = deltaRetriever.retrieveDelta(seqNum, imgNum);
if (!deltas.isEmpty()) {
- LOGGER.info("Newer delta updates are found up to sequence number: {}", curSeqNum);
+ LOGGER.info("({}) Newer delta updates are found up to sequence number {} and being sent to HDFS", retrieverType, curSeqNum);
return deltas;
}
}
// If the sequence number is < 0 or the requested delta is not available, then we
// return a full update.
- LOGGER.info("A full update is returned due to an unavailable sequence number: {}", seqNum);
- return retrieveFullImage();
+ List<K>fullImage = retrieveFullImage();
+ //Only log if we have received full image
+ if( fullImage != null && !fullImage.isEmpty()) {
+ LOGGER.info("({}) A full update is returned due to an unavailable sequence number: {}",
+ retrieverType, seqNum);
+ }
+ return fullImage;
}
private List<K> retrieveFullImage() throws Exception {
if (SentryStateBank.isEnabled(SentryServiceState.COMPONENT, SentryServiceState.FULL_UPDATE_RUNNING)){
- LOGGER.debug("A full update is being loaded. Delaying updating client with full image until its finished.");
+ LOGGER.debug("({}) A full update is being loaded. Delaying updating client with full image until its finished.", retrieverType);
return Collections.emptyList();
}
else {
http://git-wip-us.apache.org/repos/asf/sentry/blob/d4172ba9/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java
index 1318082..ee528be 100644
--- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java
+++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java
@@ -140,6 +140,7 @@ public class SentryPlugin implements SentryPolicyStorePlugin, SigUtils.SigListen
public List<PathsUpdate> getAllPathsUpdatesFrom(long pathSeqNum, long pathImgNum) throws Exception {
if (!fullUpdateNN.get()) {
// Most common case - Sentry is NOT handling a full update.
+ LOGGER.debug("Sending partial PATH update to NameNode for pathSeqNum {} and pathImgNum {}", pathSeqNum, pathImgNum);
return pathsUpdater.getAllUpdatesFrom(pathSeqNum, pathImgNum);
}
@@ -147,7 +148,7 @@ public class SentryPlugin implements SentryPolicyStorePlugin, SigUtils.SigListen
* Sentry is in the middle of signal-triggered full update.
* It already got a full update from HMS
*/
- LOGGER.info("SIGNAL HANDLING: sending full update to NameNode");
+ LOGGER.info("SIGNAL HANDLING: sending full PATH update to NameNode");
fullUpdateNN.set(false); // don't do full NN update till the next signal
List<PathsUpdate> updates =
pathsUpdater.getAllUpdatesFrom(SEQUENCE_NUMBER_UPDATE_UNINITIALIZED, IMAGE_NUMBER_UPDATE_UNINITIALIZED);
@@ -166,20 +167,21 @@ public class SentryPlugin implements SentryPolicyStorePlugin, SigUtils.SigListen
if (updates != null) {
if (!updates.isEmpty()) {
if (updates.get(0).hasFullImage()) {
- LOGGER.info("SIGNAL HANDLING: Confirmed full update to NameNode");
+ LOGGER.info("SIGNAL HANDLING: Confirmed full PATH update to NameNode for pathSeqNum {} and pathImgNum {}", pathSeqNum, pathImgNum);
} else {
- LOGGER.warn("SIGNAL HANDLING: Sending partial instead of full update to NameNode (???)");
+ LOGGER.warn("SIGNAL HANDLING: Sending partial instead of full PATH update to NameNode for pathSeqNum {} and pathImgNum {} (???)", pathSeqNum, pathImgNum);
}
} else {
- LOGGER.warn("SIGNAL HANDLING: Sending empty instead of full update to NameNode (???)");
+ LOGGER.warn("SIGNAL HANDLING: Sending empty instead of full PATH update to NameNode for pathSeqNum {} and pathImgNum {} (???)", pathSeqNum, pathImgNum);
}
} else {
- LOGGER.warn("SIGNAL HANDLING: returned NULL instead of full update to NameNode (???)");
+ LOGGER.warn("SIGNAL HANDLING: returned NULL instead of full PATH update to NameNode for pathSeqNum {} and pathImgNum {} (???)", pathSeqNum, pathImgNum);
}
return updates;
}
public List<PermissionsUpdate> getAllPermsUpdatesFrom(long permSeqNum) throws Exception {
+ LOGGER.debug("Sending partial PERM update to NameNode for permSeqNum {}", permSeqNum);
return permsUpdater.getAllUpdatesFrom(permSeqNum, UNUSED_PATH_UPDATE_IMG_NUM);
}
http://git-wip-us.apache.org/repos/asf/sentry/blob/d4172ba9/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/NotificationProcessor.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/NotificationProcessor.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/NotificationProcessor.java
index 713fc3d..07a7db4 100644
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/NotificationProcessor.java
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/NotificationProcessor.java
@@ -520,7 +520,7 @@ final class NotificationProcessor {
}
if (oldLocation.equals(newLocation)) {
- LOGGER.info(String.format("Alter partition notification ignored as"
+ LOGGER.debug(String.format("Alter partition notification ignored as"
+ "location has not changed: AuthzObj = %s, Location = %s", dbName + "."
+ "." + tableName, oldLocation));
return false;