You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sentry.apache.org by ka...@apache.org on 2018/07/25 21:20:13 UTC
sentry git commit: SENTRY-2309: Add relevant full snapshot timeline
messages to std.out. ( Arjun Mishra reviewed by Kalyan Kumar Kalvagadda and
Lina li)
Repository: sentry
Updated Branches:
refs/heads/master a3a199995 -> 3dab053e1
SENTRY-2309: Add relevant full snapshot timeline messages to std.out. ( Arjun Mishra reviewed by Kalyan Kumar Kalvagadda and Lina li)
Project: http://git-wip-us.apache.org/repos/asf/sentry/repo
Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/3dab053e
Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/3dab053e
Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/3dab053e
Branch: refs/heads/master
Commit: 3dab053e18b4197e97f4a654e2707cb1eb673047
Parents: a3a1999
Author: Kalyan Kumar Kalvagadda <kk...@cloudera.com>
Authored: Wed Jul 25 16:18:20 2018 -0500
Committer: Kalyan Kumar Kalvagadda <kk...@cloudera.com>
Committed: Wed Jul 25 16:18:20 2018 -0500
----------------------------------------------------------------------
.../sentry/hdfs/SentryAuthorizationInfo.java | 7 +++--
.../apache/sentry/hdfs/DBUpdateForwarder.java | 20 ++++++++++---
.../sentry/api/common/SentryServiceUtil.java | 12 ++++++++
.../db/service/persistent/HMSFollower.java | 30 +++++++++++++-------
.../service/thrift/FullUpdateInitializer.java | 19 ++++++++-----
.../sentry/service/thrift/SentryHMSClient.java | 10 +++++--
6 files changed, 73 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sentry/blob/3dab053e/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryAuthorizationInfo.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryAuthorizationInfo.java b/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryAuthorizationInfo.java
index 61b0d10..9dae035 100644
--- a/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryAuthorizationInfo.java
+++ b/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryAuthorizationInfo.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.util.StringUtils;
+import org.apache.sentry.api.common.SentryServiceUtil;
import org.apache.sentry.hdfs.Updateable.Update;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -195,10 +196,12 @@ public class SentryAuthorizationInfo implements Runnable {
V newUpdateable = updateable;
if (!updates.isEmpty()) {
if (updates.get(0).hasFullImage()) {
- LOG.debug(String.format("Process Update : FULL IMAGE [%s][%d][%d]",
+ String logMessage = String.format("Process Update : FULL IMAGE [%s][%d][%d]",
newUpdateable.getClass().getSimpleName(),
updates.get(0).getSeqNum(),
- updates.get(0).getImgNum()));
+ updates.get(0).getImgNum());
+ LOG.info(logMessage);
+ System.out.println(SentryServiceUtil.getCurrentTimeStampWithMessage(logMessage));
newUpdateable = (V)newUpdateable.updateFull(updates.remove(0));
}
// Any more elements ?
http://git-wip-us.apache.org/repos/asf/sentry/blob/3dab053e/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/DBUpdateForwarder.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/DBUpdateForwarder.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/DBUpdateForwarder.java
index 065adb7..d47b9f7 100644
--- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/DBUpdateForwarder.java
+++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/DBUpdateForwarder.java
@@ -25,6 +25,7 @@ import java.util.List;
import java.util.Map;
import javax.annotation.concurrent.ThreadSafe;
+import org.apache.sentry.api.common.SentryServiceUtil;
import org.apache.sentry.core.model.db.AccessConstants;
import org.apache.sentry.hdfs.service.thrift.TPrivilegeEntity;
import org.apache.sentry.service.thrift.SentryServiceState;
@@ -93,11 +94,17 @@ class DBUpdateForwarder<K extends Updateable.Update> {
if (curImgNum > imgNum) {
// In case a new HMS snapshot has been processed, then return a full paths image.
+ String logMessage = String.format("(%s) Current image num %d is greater than requested "
+ + "image num %d. Request a full update", retrieverType, curImgNum, imgNum);
+ LOGGER.info(logMessage);
+ System.out.println(SentryServiceUtil.getCurrentTimeStampWithMessage(logMessage));
List<K>fullImage = retrieveFullImage();
//Only log if we have received full image
if( !fullImage.isEmpty()) {
- LOGGER.info("({}) A newer full update with image number {} was found and being sent to HDFS",
- retrieverType, curImgNum);
+ logMessage = String.format("(%s) A newer full update with image number %d "
+ + "is less than requested image number %d. Send full update to HDFS", retrieverType, curImgNum, imgNum);
+ LOGGER.info(logMessage);
+ System.out.println(SentryServiceUtil.getCurrentTimeStampWithMessage(logMessage));
}
return fullImage;
}
@@ -128,11 +135,16 @@ class DBUpdateForwarder<K extends Updateable.Update> {
// If the sequence number is < 0 or the requested delta is not available, then we
// return a full update.
+ String logMessage = String.format("(%s) Requested sequence number %d is less than 0 or "
+ + "requested deltas for that sequence number are not available. Fetch a full update", retrieverType, seqNum);
+ LOGGER.info(logMessage);
+ System.out.println(SentryServiceUtil.getCurrentTimeStampWithMessage(logMessage));
List<K>fullImage = retrieveFullImage();
//Only log if we have received full image
if( fullImage != null && !fullImage.isEmpty()) {
- LOGGER.info("({}) A full update is returned due to an unavailable sequence number: {}",
- retrieverType, seqNum);
+ logMessage = String.format("(%s) A full update is returned due to an unavailable sequence number: %d", retrieverType, seqNum);
+ LOGGER.info(logMessage);
+ System.out.println(SentryServiceUtil.getCurrentTimeStampWithMessage(logMessage));
}
return fullImage;
}
http://git-wip-us.apache.org/repos/asf/sentry/blob/3dab053e/sentry-service/sentry-service-api/src/main/java/org/apache/sentry/api/common/SentryServiceUtil.java
----------------------------------------------------------------------
diff --git a/sentry-service/sentry-service-api/src/main/java/org/apache/sentry/api/common/SentryServiceUtil.java b/sentry-service/sentry-service-api/src/main/java/org/apache/sentry/api/common/SentryServiceUtil.java
index 719e616..8cdbde4 100644
--- a/sentry-service/sentry-service-api/src/main/java/org/apache/sentry/api/common/SentryServiceUtil.java
+++ b/sentry-service/sentry-service-api/src/main/java/org/apache/sentry/api/common/SentryServiceUtil.java
@@ -19,7 +19,9 @@
package org.apache.sentry.api.common;
+import java.text.SimpleDateFormat;
import java.util.Arrays;
+import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -319,4 +321,14 @@ public final class SentryServiceUtil {
private static boolean isNULL(String s) {
return Strings.isNullOrEmpty(s) || s.equals("__NULL__");
}
+
+ public static String getCurrentTimeStampWithMessage(String logMessage) {
+ String date = new SimpleDateFormat("yyyy.MM.dd.HH.mm.ss").format(new Date());
+
+ if(StringUtils.isBlank(logMessage)) {
+ return date;
+ }
+
+ return String.format("%s: %s", date, logMessage);
+ }
}
http://git-wip-us.apache.org/repos/asf/sentry/blob/3dab053e/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/provider/db/service/persistent/HMSFollower.java
----------------------------------------------------------------------
diff --git a/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/provider/db/service/persistent/HMSFollower.java b/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/provider/db/service/persistent/HMSFollower.java
index 7667c47..fb826cf 100644
--- a/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/provider/db/service/persistent/HMSFollower.java
+++ b/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/provider/db/service/persistent/HMSFollower.java
@@ -273,16 +273,19 @@ public class HMSFollower implements Runnable, AutoCloseable, PubSub.Subscriber {
*/
private boolean isFullSnapshotRequired(long latestSentryNotificationId) throws Exception {
if (sentryStore.isHmsNotificationEmpty()) {
- LOGGER.debug("Sentry Store has no HMS Notifications. Create Full HMS Snapshot. "
- + "latest sentry notification Id = {}", latestSentryNotificationId);
+ String logMessage = String.format("Sentry Store has no HMS Notifications. Create Full HMS Snapshot. "
+ + "latest sentry notification Id = %d", latestSentryNotificationId);
+ LOGGER.debug(logMessage);
+ System.out.println(SentryServiceUtil.getCurrentTimeStampWithMessage(logMessage));
return true;
}
// Once HDFS sync is enabled, and if MAuthzPathsMapping
// table is still empty, we need to request a full snapshot
if (sentryStore.isAuthzPathsSnapshotEmpty()) {
- LOGGER.debug("HDFSSync is enabled and MAuthzPathsMapping table is empty." +
- " Need to request a full snapshot");
+ String logMessage = String.format("HDFSSync is enabled and MAuthzPathsMapping table is empty. Need to request a full snapshot", latestSentryNotificationId);
+ LOGGER.debug(logMessage);
+ System.out.println(SentryServiceUtil.getCurrentTimeStampWithMessage(logMessage));
return true;
}
@@ -293,10 +296,11 @@ public class HMSFollower implements Runnable, AutoCloseable, PubSub.Subscriber {
// check if forced full update is required, reset update flag to false
// to only do it once per forced full update request.
if (fullUpdateHMS.compareAndSet(true, false)) {
- LOGGER.info(FULL_UPDATE_TRIGGER + "initiating full HMS snapshot request");
+ String logMessage = FULL_UPDATE_TRIGGER + "initiating full HMS snapshot request";
+ LOGGER.info(logMessage);
+ System.out.println(SentryServiceUtil.getCurrentTimeStampWithMessage(logMessage));
return true;
}
-
return false;
}
@@ -357,8 +361,10 @@ public class HMSFollower implements Runnable, AutoCloseable, PubSub.Subscriber {
long firstNotificationId = eventList.get(0).getEventId();
if (firstNotificationId > (latestProcessedId + 1)) {
- LOGGER.info("First HMS event notification Id = {} is greater than latest Sentry processed"
- + "notification Id = {} + 1. Need to request a full HMS snapshot.", firstNotificationId, latestProcessedId);
+ String logMessage = String.format("First HMS event notification Id = %d is greater than latest Sentry processed"
+ + "notification Id = %d + 1. Need to request a full HMS snapshot.", firstNotificationId, latestProcessedId);
+ LOGGER.info(logMessage);
+ System.out.println(SentryServiceUtil.getCurrentTimeStampWithMessage(logMessage));
return true;
}
@@ -395,7 +401,9 @@ public class HMSFollower implements Runnable, AutoCloseable, PubSub.Subscriber {
}
try {
if (hdfsSyncEnabled) {
- LOGGER.info("Persisting full snapshot for notification Id = {}", snapshotInfo.getId());
+ String logMessage = String.format("Persisting full snapshot for notification Id = %d", snapshotInfo.getId());
+ LOGGER.info(logMessage);
+ System.out.println(SentryServiceUtil.getCurrentTimeStampWithMessage(logMessage));
sentryStore.persistFullPathsImage(snapshotInfo.getPathImage(), snapshotInfo.getId());
} else {
// We need to persist latest notificationID for next poll
@@ -412,7 +420,9 @@ public class HMSFollower implements Runnable, AutoCloseable, PubSub.Subscriber {
wakeUpWaitingClientsForSync(snapshotInfo.getId());
// HMSFollower connected to HMS and it finished full snapshot if that was required
// Log this message only once
- LOGGER.info("Sentry HMS support is ready");
+ String logMessage = String.format("Create full snapshot process is complete: snapshot Id %d", snapshotInfo.getId());
+ LOGGER.info(logMessage);
+ System.out.println(SentryServiceUtil.getCurrentTimeStampWithMessage(logMessage));
return snapshotInfo.getId();
} catch(Exception failure) {
LOGGER.error("Received exception while creating HMS path full snapshot ");
http://git-wip-us.apache.org/repos/asf/sentry/blob/3dab053e/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/service/thrift/FullUpdateInitializer.java
----------------------------------------------------------------------
diff --git a/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/service/thrift/FullUpdateInitializer.java b/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/service/thrift/FullUpdateInitializer.java
index 992d8ab..3e27d1b 100644
--- a/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/service/thrift/FullUpdateInitializer.java
+++ b/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/service/thrift/FullUpdateInitializer.java
@@ -299,15 +299,20 @@ public final class FullUpdateInitializer implements AutoCloseable {
}
LOGGER.debug("Fetched partitions for db = {}, table = {}",
- dbName, tblName);
-
+ dbName, tblName);
Collection<String> partitionNames = new ArrayList<>(tblParts.size());
- for (Partition part : tblParts) {
- String partPath = pathFromURI(part.getSd().getLocation());
- if (partPath != null) {
- partitionNames.add(partPath.intern());
+
+ for (Partition part : tblParts) {
+ try {
+ String partPath = pathFromURI(part.getSd().getLocation());
+ if (partPath != null) {
+ partitionNames.add(partPath.intern());
+ }
+ } catch(Exception e) {
+ LOGGER.error("Exception while fetching partitions for db = {} table = {}", dbName, tblName);
+ throw e;
+ }
}
- }
return new ObjectMapping(authName, partitionNames);
}
}
http://git-wip-us.apache.org/repos/asf/sentry/blob/3dab053e/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/service/thrift/SentryHMSClient.java
----------------------------------------------------------------------
diff --git a/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/service/thrift/SentryHMSClient.java b/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/service/thrift/SentryHMSClient.java
index 12266cb..cfb0d78 100644
--- a/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/service/thrift/SentryHMSClient.java
+++ b/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/service/thrift/SentryHMSClient.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.NotificationEvent;
import org.apache.hadoop.hive.metastore.api.NotificationEventResponse;
import org.apache.hadoop.hive.metastore.messaging.MessageDeserializer;
+import org.apache.sentry.api.common.SentryServiceUtil;
import org.apache.sentry.binding.metastore.messaging.json.SentryJSONMessageDeserializer;
import org.apache.sentry.core.common.utils.SentryConstants;
import org.apache.sentry.provider.db.service.persistent.PathsImage;
@@ -242,12 +243,17 @@ public class SentryHMSClient implements AutoCloseable {
* corresponding to that name.
*/
private Map<String, Collection<String>> fetchFullUpdate() throws Exception{
- LOGGER.info("Request full HMS snapshot");
+ String logMessage = "Request full HMS snapshot";
+ LOGGER.info(logMessage);
+ System.out.println(SentryServiceUtil.getCurrentTimeStampWithMessage(logMessage));
+
try (FullUpdateInitializer updateInitializer =
new FullUpdateInitializer(hiveConnectionFactory, conf);
Context context = updateTimer.time()) {
Map<String, Collection<String>> pathsUpdate = updateInitializer.getFullHMSSnapshot();
- LOGGER.info("Obtained full HMS snapshot");
+ logMessage = "Obtained full HMS snapshot";
+ LOGGER.info(logMessage);
+ System.out.println(SentryServiceUtil.getCurrentTimeStampWithMessage(logMessage));
return pathsUpdate;
} catch (Exception exception) {
failedSnapshotsCount.inc();