You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sentry.apache.org by ls...@apache.org on 2015/11/03 07:55:29 UTC
incubator-sentry git commit: SENTRY-510: Metrics collection for
Sentry HDFS plugin (Li Li via Lenni Kuff)
Repository: incubator-sentry
Updated Branches:
refs/heads/master cccebe099 -> 2bd258105
SENTRY-510: Metrics collection for Sentry HDFS plugin (Li Li via Lenni Kuff)
Change-Id: Ied49f6872265e2d6319f3bc6b911e8278a1662d0
Project: http://git-wip-us.apache.org/repos/asf/incubator-sentry/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-sentry/commit/2bd25810
Tree: http://git-wip-us.apache.org/repos/asf/incubator-sentry/tree/2bd25810
Diff: http://git-wip-us.apache.org/repos/asf/incubator-sentry/diff/2bd25810
Branch: refs/heads/master
Commit: 2bd258105eb5395a83b14b6830f975342b8c1333
Parents: cccebe0
Author: Lenni Kuff <ls...@cloudera.com>
Authored: Mon Nov 2 22:54:50 2015 -0800
Committer: Lenni Kuff <ls...@cloudera.com>
Committed: Mon Nov 2 22:55:01 2015 -0800
----------------------------------------------------------------------
.../org/apache/sentry/hdfs/MetastorePlugin.java | 11 ++
.../apache/sentry/hdfs/PluginCacheSyncUtil.java | 14 ++-
.../sentry/hdfs/SentryHDFSServiceProcessor.java | 23 ++++-
.../sentry/hdfs/SentryHdfsMetricsUtil.java | 101 +++++++++++++++++++
.../org/apache/sentry/hdfs/SentryPlugin.java | 8 ++
.../db/service/thrift/SentryMetrics.java | 23 +++++
6 files changed, 177 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/2bd25810/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/MetastorePlugin.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/MetastorePlugin.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/MetastorePlugin.java
index 8abdc83..f88295d 100644
--- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/MetastorePlugin.java
+++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/MetastorePlugin.java
@@ -31,6 +31,7 @@ import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
+import com.codahale.metrics.Timer;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler;
@@ -282,10 +283,15 @@ public class MetastorePlugin extends SentryMetastoreListenerPlugin {
}
protected void notifySentryNoLock(PathsUpdate update) {
+ final Timer.Context timerContext =
+ SentryHdfsMetricsUtil.getNotifyHMSUpdateTimer.time();
try {
getClient().notifyHMSUpdate(update);
} catch (Exception e) {
LOGGER.error("Could not send update to Sentry HDFS Service !!", e);
+ SentryHdfsMetricsUtil.getFailedNotifyHMSUpdateCounter.inc();
+ } finally {
+ timerContext.stop();
}
}
@@ -304,7 +310,12 @@ public class MetastorePlugin extends SentryMetastoreListenerPlugin {
}
protected void applyLocal(PathsUpdate update) {
+ final Timer.Context timerContext =
+ SentryHdfsMetricsUtil.getApplyLocalUpdateTimer.time();
authzPaths.updatePartial(Lists.newArrayList(update), new ReentrantReadWriteLock());
+ timerContext.stop();
+ SentryHdfsMetricsUtil.getApplyLocalUpdateHistogram.update(
+ update.getPathChanges().size());
}
private void notifySentryAndApplyLocal(PathsUpdate update) {
http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/2bd25810/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PluginCacheSyncUtil.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PluginCacheSyncUtil.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PluginCacheSyncUtil.java
index e297286..5e2f98e 100644
--- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PluginCacheSyncUtil.java
+++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PluginCacheSyncUtil.java
@@ -22,6 +22,7 @@ import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
+import com.codahale.metrics.Timer;
import org.apache.curator.framework.recipes.atomic.DistributedAtomicLong;
import org.apache.curator.framework.recipes.cache.PathChildrenCache;
import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent;
@@ -118,6 +119,7 @@ public class PluginCacheSyncUtil {
}
public void handleCacheUpdate(Update update) throws SentryPluginException {
+ final Timer.Context timerContext = SentryHdfsMetricsUtil.getCacheSyncToZKTimer.time();
// post message to ZK cache
try {
// Acquire ZK lock for update cache sync. This ensures that the counter
@@ -127,10 +129,13 @@ public class PluginCacheSyncUtil {
"Failed to get ZK lock for update cache syncup");
}
} catch (Exception e1) {
+ // Stop timer in advance
+ timerContext.stop();
+ SentryHdfsMetricsUtil.getFailedCacheSyncToZK.inc();
throw new SentryPluginException(
"Error getting ZK lock for update cache syncup" + e1, e1);
}
-
+ boolean failed = false;
try {
try {
// increment the global sequence counter if this is not a full update
@@ -142,6 +147,7 @@ public class PluginCacheSyncUtil {
}
}
} catch (Exception e1) {
+ failed = true;
throw new SentryPluginException(
"Error setting ZK counter for update cache syncup" + e1, e1);
}
@@ -154,6 +160,7 @@ public class PluginCacheSyncUtil {
haContext.getCuratorFramework().create().creatingParentsIfNeeded()
.forPath(newPath, update.serialize());
} catch (Exception e) {
+ failed = true;
throw new SentryPluginException("error posting update to ZK ", e);
}
} finally {
@@ -161,9 +168,14 @@ public class PluginCacheSyncUtil {
try {
updatorLock.release();
} catch (Exception e) {
+ // Stop timer in advance
+ timerContext.stop();
+ SentryHdfsMetricsUtil.getFailedCacheSyncToZK.inc();
throw new SentryPluginException(
"Error releasing ZK lock for update cache syncup" + e, e);
}
+ timerContext.stop();
+ if (failed) SentryHdfsMetricsUtil.getFailedCacheSyncToZK.inc();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/2bd25810/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHDFSServiceProcessor.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHDFSServiceProcessor.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHDFSServiceProcessor.java
index 80f3648..e4f3f58 100644
--- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHDFSServiceProcessor.java
+++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHDFSServiceProcessor.java
@@ -22,6 +22,7 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import com.codahale.metrics.Timer;
import org.apache.sentry.hdfs.service.thrift.SentryHDFSService;
import org.apache.sentry.hdfs.service.thrift.TAuthzUpdateResponse;
import org.apache.sentry.hdfs.service.thrift.TPathsUpdate;
@@ -45,9 +46,15 @@ public class SentryHDFSServiceProcessor implements SentryHDFSService.Iface {
throw new TException(
"This Sentry server is not communicating with other nodes and out of sync ");
}
- List<PermissionsUpdate> permUpdates = SentryPlugin.instance.getAllPermsUpdatesFrom(permSeqNum);
- List<PathsUpdate> pathUpdates = SentryPlugin.instance.getAllPathsUpdatesFrom(pathSeqNum);
+ final Timer.Context timerContext =
+ SentryHdfsMetricsUtil.getAllAuthzUpdatesTimer.time();
try {
+ List<PermissionsUpdate> permUpdates =
+ SentryPlugin.instance.getAllPermsUpdatesFrom(permSeqNum);
+ SentryHdfsMetricsUtil.getPermUpdateHistogram.update(permUpdates.size());
+ List<PathsUpdate> pathUpdates =
+ SentryPlugin.instance.getAllPathsUpdatesFrom(pathSeqNum);
+ SentryHdfsMetricsUtil.getPathUpdateHistogram.update(pathUpdates.size());
for (PathsUpdate update : pathUpdates) {
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("### Sending PATH preUpdate seq [" + update.getSeqNum() + "] ###");
@@ -80,6 +87,8 @@ public class SentryHDFSServiceProcessor implements SentryHDFSService.Iface {
} catch (Exception e) {
LOGGER.error("Error Sending updates to downstream Cache", e);
throw new TException(e);
+ } finally {
+ timerContext.stop();
}
} else {
LOGGER.error("SentryPlugin not initialized yet !!");
@@ -90,6 +99,8 @@ public class SentryHDFSServiceProcessor implements SentryHDFSService.Iface {
@Override
public void handle_hms_notification(TPathsUpdate update) throws TException {
+ final Timer.Context timerContext =
+ SentryHdfsMetricsUtil.getHandleHmsNotificationTimer.time();
try {
PathsUpdate hmsUpdate = new PathsUpdate(update);
if (SentryPlugin.instance != null) {
@@ -100,7 +111,15 @@ public class SentryHDFSServiceProcessor implements SentryHDFSService.Iface {
}
} catch (Exception e) {
LOGGER.error("Error handling notification from HMS", e);
+ SentryHdfsMetricsUtil.getFailedHandleHmsNotificationCounter.inc();
throw new TException(e);
+ } finally {
+ timerContext.stop();
+ SentryHdfsMetricsUtil.getHandleHmsPathChangeHistogram.update(
+ update.getPathChangesSize());
+ if (update.isHasFullImage()) {
+ SentryHdfsMetricsUtil.getHandleHmsHasFullImageCounter.inc();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/2bd25810/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHdfsMetricsUtil.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHdfsMetricsUtil.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHdfsMetricsUtil.java
new file mode 100644
index 0000000..b67c94a
--- /dev/null
+++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHdfsMetricsUtil.java
@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.hdfs;
+
+import com.codahale.metrics.Counter;
+import com.codahale.metrics.Histogram;
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.Timer;
+import org.apache.sentry.provider.db.service.thrift.SentryMetrics;
+
+/**
+ * Util class to support metrics.
+ */
+public class SentryHdfsMetricsUtil {
+ // SentryMetrics
+ private static final SentryMetrics sentryMetrics = SentryMetrics.getInstance();
+
+ // Metrics for get_all_authz_updates_from in SentryHDFSServiceProcessor
+ // The time used for each get_all_authz_updates_from
+ public static final Timer getAllAuthzUpdatesTimer = sentryMetrics.getTimer(
+ MetricRegistry.name(SentryHDFSServiceProcessor.class,
+ "get-all-authz-updates-from"));
+ // The size of perm updates for each get_all_authz_updates_from
+ public static final Histogram getPermUpdateHistogram = sentryMetrics.getHistogram(
+ MetricRegistry.name(SentryHDFSServiceProcessor.class, "perm-updates-size"));
+ // The size of path updates for each get_all_authz_updates_from
+ public static final Histogram getPathUpdateHistogram = sentryMetrics.getHistogram(
+ MetricRegistry.name(SentryHDFSServiceProcessor.class, "paths-updates-size"));
+
+ // Metrics for handle_hms_notification in SentryHDFSServiceProcessor
+ // The time used for each handle_hms_notification
+ public static final Timer getHandleHmsNotificationTimer = sentryMetrics.getTimer(
+ MetricRegistry.name(SentryHDFSServiceProcessor.class, "handle-hms-notification"));
+ // The number of failed handle_hms_notification
+ public static final Counter getFailedHandleHmsNotificationCounter =
+ sentryMetrics.getCounter(MetricRegistry.name(SentryHDFSServiceProcessor.class,
+ "handle-hms-notification", "failed-num"));
+ // The number of handle_hms_notification with full image update
+ public static final Counter getHandleHmsHasFullImageCounter = sentryMetrics.getCounter(
+ MetricRegistry.name(SentryHDFSServiceProcessor.class, "handle-hms-notification",
+ "has-full-image-num"));
+ // The size of path changes for each handle_hms_notification
+ public static final Histogram getHandleHmsPathChangeHistogram = sentryMetrics.getHistogram(
+ MetricRegistry.name(SentryHDFSServiceProcessor.class, "handle-hms-notification",
+ "path-changes-size"));
+
+ // Metrics for retrieveFullImage in SentryPlugin.PermImageRetriever
+ // The time used for each retrieveFullImage
+ public static final Timer getRetrieveFullImageTimer = sentryMetrics.getTimer(
+ MetricRegistry.name(SentryPlugin.PermImageRetriever.class, "retrieve-full-image"));
+ // The size of privilege changes for each retrieveFullImage
+ public static final Histogram getPrivilegeChangesHistogram = sentryMetrics.getHistogram(
+ MetricRegistry.name(SentryPlugin.PermImageRetriever.class, "retrieve-full-image",
+ "privilege-changes-size"));
+ // The size of role changes for each retrieveFullImage call
+ public static final Histogram getRoleChangesHistogram = sentryMetrics.getHistogram(
+ MetricRegistry.name(SentryPlugin.PermImageRetriever.class, "retrieve-full-image",
+ "role-changes-size"));
+
+ // Metrics for notifySentry HMS update in MetaStorePlugin
+ // The timer used for each notifySentry
+ public static final Timer getNotifyHMSUpdateTimer = sentryMetrics.getTimer(
+ MetricRegistry.name(MetastorePlugin.class, "notify-sentry-HMS-update"));
+ // The number of failed notifySentry
+ public static final Counter getFailedNotifyHMSUpdateCounter = sentryMetrics.getCounter(
+ MetricRegistry.name(MetastorePlugin.class, "notify-sentry-HMS-update",
+ "failed-num"));
+
+ // Metrics for applyLocal update in MetastorePlugin
+ // The time used for each applyLocal
+ public static final Timer getApplyLocalUpdateTimer = sentryMetrics.getTimer(
+ MetricRegistry.name(MetastorePlugin.class, "apply-local-update"));
+ // The size of path changes for each applyLocal
+ public static final Histogram getApplyLocalUpdateHistogram = sentryMetrics.getHistogram(
+ MetricRegistry.name(MetastorePlugin.class, "apply-local-update",
+ "path-change-size"));
+
+ // Metrics for handleCacheUpdate to ZK in PluginCacheSyncUtil
+ // The time used for each handleCacheUpdate
+ public static final Timer getCacheSyncToZKTimer = sentryMetrics.getTimer(
+ MetricRegistry.name(PluginCacheSyncUtil.class, "cache-sync-to-zk"));
+ // The number of failed handleCacheUpdate
+ public static final Counter getFailedCacheSyncToZK = sentryMetrics.getCounter(
+ MetricRegistry.name(PluginCacheSyncUtil.class, "cache-sync-to-zk", "failed-num"));
+}
http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/2bd25810/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java
index 93514e6..647e8fc 100644
--- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java
+++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java
@@ -24,6 +24,7 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
+import com.codahale.metrics.Timer;
import org.apache.hadoop.conf.Configuration;
import org.apache.sentry.hdfs.ServiceConstants.ServerConfig;
import org.apache.sentry.hdfs.UpdateForwarder.ExternalImageRetriever;
@@ -66,6 +67,8 @@ public class SentryPlugin implements SentryPolicyStorePlugin {
@Override
public PermissionsUpdate retrieveFullImage(long currSeqNum) {
+ final Timer.Context timerContext =
+ SentryHdfsMetricsUtil.getRetrieveFullImageTimer.time();
Map<String, HashMap<String, String>> privilegeImage = sentryStore.retrieveFullPrivilegeImage();
Map<String, LinkedList<String>> roleImage = sentryStore.retrieveFullRoleImage();
@@ -85,6 +88,11 @@ public class SentryPlugin implements SentryPolicyStorePlugin {
}
PermissionsUpdate permissionsUpdate = new PermissionsUpdate(tPermUpdate);
permissionsUpdate.setSeqNum(currSeqNum);
+ timerContext.stop();
+ SentryHdfsMetricsUtil.getPrivilegeChangesHistogram.update(
+ tPermUpdate.getPrivilegeChangesSize());
+ SentryHdfsMetricsUtil.getRoleChangesHistogram.update(
+ tPermUpdate.getRoleChangesSize());
return permissionsUpdate;
}
http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/2bd25810/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryMetrics.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryMetrics.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryMetrics.java
index 55bec0b..6eb00a1 100644
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryMetrics.java
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryMetrics.java
@@ -18,7 +18,9 @@
package org.apache.sentry.provider.db.service.thrift;
import com.codahale.metrics.ConsoleReporter;
+import com.codahale.metrics.Counter;
import com.codahale.metrics.Gauge;
+import com.codahale.metrics.Histogram;
import com.codahale.metrics.JmxReporter;
import com.codahale.metrics.Metric;
import com.codahale.metrics.MetricRegistry;
@@ -69,6 +71,27 @@ public class SentryMetrics {
public final Timer listPrivilegesByAuthorizableTimer = SentryMetricsServletContextListener.METRIC_REGISTRY.timer(
MetricRegistry.name(SentryPolicyStoreProcessor.class, "list-privileges-by-authorizable"));
+ /**
+ * Return a Timer with name.
+ */
+ public final Timer getTimer(String name) {
+ return SentryMetricsServletContextListener.METRIC_REGISTRY.timer(name);
+ }
+
+ /**
+ * Return a Histogram with name.
+ */
+ public final Histogram getHistogram(String name) {
+ return SentryMetricsServletContextListener.METRIC_REGISTRY.histogram(name);
+ }
+
+ /**
+ * Return a Counter with name.
+ */
+ public final Counter getCounter(String name) {
+ return SentryMetricsServletContextListener.METRIC_REGISTRY.counter(name);
+ }
+
private SentryMetrics() {
registerMetricSet("gc", new GarbageCollectorMetricSet(), SentryMetricsServletContextListener.METRIC_REGISTRY);
registerMetricSet("buffers", new BufferPoolMetricSet(ManagementFactory.getPlatformMBeanServer()),