You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sentry.apache.org by ak...@apache.org on 2017/04/04 22:41:42 UTC
sentry git commit: SENTRY-1677: Add metrics to measure how much time
to get Delta Path/Perm Updates (Alex Kolabsov, reviewed by Na Li)
Repository: sentry
Updated Branches:
refs/heads/sentry-ha-redesign 15613a1cc -> 9739dd80a
SENTRY-1677: Add metrics to measure how much time to get Delta Path/Perm Updates (Alex Kolabsov, reviewed by Na Li)
Project: http://git-wip-us.apache.org/repos/asf/sentry/repo
Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/9739dd80
Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/9739dd80
Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/9739dd80
Branch: refs/heads/sentry-ha-redesign
Commit: 9739dd80a1635dc07f71f05857b5d3278872d1ca
Parents: 15613a1
Author: Alexander Kolbasov <ak...@cloudera.com>
Authored: Tue Apr 4 15:40:46 2017 -0700
Committer: Alexander Kolbasov <ak...@cloudera.com>
Committed: Tue Apr 4 15:40:46 2017 -0700
----------------------------------------------------------------------
.../apache/sentry/hdfs/PathDeltaRetriever.java | 45 +++++++++++---------
.../apache/sentry/hdfs/PermDeltaRetriever.java | 41 ++++++++++--------
.../sentry/hdfs/SentryHdfsMetricsUtil.java | 37 ++++++++++++----
3 files changed, 78 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sentry/blob/9739dd80/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PathDeltaRetriever.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PathDeltaRetriever.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PathDeltaRetriever.java
index cea5b9d..5425114 100644
--- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PathDeltaRetriever.java
+++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PathDeltaRetriever.java
@@ -17,6 +17,7 @@
*/
package org.apache.sentry.hdfs;
+import com.codahale.metrics.Timer;
import org.apache.sentry.provider.db.service.model.MSentryPathChange;
import org.apache.sentry.provider.db.service.persistent.SentryStore;
@@ -43,34 +44,40 @@ public class PathDeltaRetriever implements DeltaRetriever<PathsUpdate> {
@Override
public Collection<PathsUpdate> retrieveDelta(long seqNum) throws Exception {
- Collection<MSentryPathChange> mSentryPathChanges =
- sentryStore.getMSentryPathChanges(seqNum);
- if (mSentryPathChanges.isEmpty()) {
- return Collections.emptyList();
- }
+ try (final Timer.Context timerContext =
+ SentryHdfsMetricsUtil.getDeltaPathChangesTimer.time()) {
+ Collection<MSentryPathChange> mSentryPathChanges =
+ sentryStore.getMSentryPathChanges(seqNum);
+
+ SentryHdfsMetricsUtil.getDeltaPathChangesHistogram.update(mSentryPathChanges.size());
+
+ if (mSentryPathChanges.isEmpty()) {
+ return Collections.emptyList();
+ }
- Collection<PathsUpdate> updates = new ArrayList<>(mSentryPathChanges.size());
- for (MSentryPathChange mSentryPathChange : mSentryPathChanges) {
- // Gets the changeID from the persisted MSentryPathChange.
- long changeID = mSentryPathChange.getChangeID();
- // Creates a corresponding PathsUpdate and deserialize the
- // persisted delta update in JSON format to TPathsUpdate with
- // associated changeID.
- PathsUpdate pathsUpdate = new PathsUpdate();
- pathsUpdate.JSONDeserialize(mSentryPathChange.getPathChange());
- pathsUpdate.setSeqNum(changeID);
- updates.add(pathsUpdate);
+ Collection<PathsUpdate> updates = new ArrayList<>(mSentryPathChanges.size());
+ for (MSentryPathChange mSentryPathChange : mSentryPathChanges) {
+ // Gets the changeID from the persisted MSentryPathChange.
+ long changeID = mSentryPathChange.getChangeID();
+ // Creates a corresponding PathsUpdate and deserialize the
+ // persisted delta update in JSON format to TPathsUpdate with
+ // associated changeID.
+ PathsUpdate pathsUpdate = new PathsUpdate();
+ pathsUpdate.JSONDeserialize(mSentryPathChange.getPathChange());
+ pathsUpdate.setSeqNum(changeID);
+ updates.add(pathsUpdate);
+ }
+ return updates;
}
- return updates;
}
@Override
- public boolean isDeltaAvailable(long seqNum) throws Exception {
+ public boolean isDeltaAvailable ( long seqNum) throws Exception {
return sentryStore.pathChangeExists(seqNum);
}
@Override
- public long getLatestDeltaID() throws Exception {
+ public long getLatestDeltaID () throws Exception {
return sentryStore.getLastProcessedPathChangeID();
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/sentry/blob/9739dd80/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PermDeltaRetriever.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PermDeltaRetriever.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PermDeltaRetriever.java
index 9649b02..a29fc74 100644
--- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PermDeltaRetriever.java
+++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PermDeltaRetriever.java
@@ -17,6 +17,7 @@
*/
package org.apache.sentry.hdfs;
+import com.codahale.metrics.Timer;
import org.apache.sentry.provider.db.service.model.MSentryPermChange;
import org.apache.sentry.provider.db.service.persistent.SentryStore;
@@ -43,25 +44,31 @@ public class PermDeltaRetriever implements DeltaRetriever<PermissionsUpdate> {
@Override
public Collection<PermissionsUpdate> retrieveDelta(long seqNum) throws Exception {
- Collection<MSentryPermChange> mSentryPermChanges =
- sentryStore.getMSentryPermChanges(seqNum);
- if (mSentryPermChanges.isEmpty()) {
- return Collections.emptyList();
- }
+ try (final Timer.Context timerContext =
+ SentryHdfsMetricsUtil.getDeltaPermChangesTimer.time()) {
+ Collection<MSentryPermChange> mSentryPermChanges =
+ sentryStore.getMSentryPermChanges(seqNum);
+
+ SentryHdfsMetricsUtil.getDeltaPermChangesHistogram.update(mSentryPermChanges.size());
+
+ if (mSentryPermChanges.isEmpty()) {
+ return Collections.emptyList();
+ }
- Collection<PermissionsUpdate> updates = new ArrayList<>(mSentryPermChanges.size());
- for (MSentryPermChange mSentryPermChange : mSentryPermChanges) {
- // Get the changeID from the persisted MSentryPermChange
- long changeID = mSentryPermChange.getChangeID();
- // Create a corresponding PermissionsUpdate and deserialize the
- // persisted delta update in JSON format to TPermissionsUpdate with
- // associated changeID.
- PermissionsUpdate permsUpdate = new PermissionsUpdate();
- permsUpdate.JSONDeserialize(mSentryPermChange.getPermChange());
- permsUpdate.setSeqNum(changeID);
- updates.add(permsUpdate);
+ Collection<PermissionsUpdate> updates = new ArrayList<>(mSentryPermChanges.size());
+ for (MSentryPermChange mSentryPermChange : mSentryPermChanges) {
+ // Get the changeID from the persisted MSentryPermChange
+ long changeID = mSentryPermChange.getChangeID();
+ // Create a corresponding PermissionsUpdate and deserialize the
+ // persisted delta update in JSON format to TPermissionsUpdate with
+ // associated changeID.
+ PermissionsUpdate permsUpdate = new PermissionsUpdate();
+ permsUpdate.JSONDeserialize(mSentryPermChange.getPermChange());
+ permsUpdate.setSeqNum(changeID);
+ updates.add(permsUpdate);
+ }
+ return updates;
}
- return updates;
}
@Override
http://git-wip-us.apache.org/repos/asf/sentry/blob/9739dd80/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHdfsMetricsUtil.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHdfsMetricsUtil.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHdfsMetricsUtil.java
index 28bf20e..932a5c0 100644
--- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHdfsMetricsUtil.java
+++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHdfsMetricsUtil.java
@@ -27,20 +27,20 @@ import org.apache.sentry.provider.db.service.thrift.SentryMetrics;
/**
* Util class to support metrics.
*/
-public class SentryHdfsMetricsUtil {
+final class SentryHdfsMetricsUtil {
// SentryMetrics
private static final SentryMetrics sentryMetrics = SentryMetrics.getInstance();
// Metrics for get_all_authz_updates_from in SentryHDFSServiceProcessor
// The time used for each get_all_authz_updates_from
- public static final Timer getAllAuthzUpdatesTimer = sentryMetrics.getTimer(
+ static final Timer getAllAuthzUpdatesTimer = sentryMetrics.getTimer(
MetricRegistry.name(SentryHDFSServiceProcessor.class,
"get-all-authz-updates-from"));
// The size of perm updates for each get_all_authz_updates_from
- public static final Histogram getPermUpdateHistogram = sentryMetrics.getHistogram(
+ static final Histogram getPermUpdateHistogram = sentryMetrics.getHistogram(
MetricRegistry.name(SentryHDFSServiceProcessor.class, "perm-updates-size"));
// The size of path updates for each get_all_authz_updates_from
- public static final Histogram getPathUpdateHistogram = sentryMetrics.getHistogram(
+ static final Histogram getPathUpdateHistogram = sentryMetrics.getHistogram(
MetricRegistry.name(SentryHDFSServiceProcessor.class, "paths-updates-size"));
// Metrics for handle_hms_notification in SentryHDFSServiceProcessor
@@ -62,27 +62,46 @@ public class SentryHdfsMetricsUtil {
// Metrics for retrievePermFullImage in PermImageRetriever
// The time used for each retrievePermFullImage
- public static final Timer getRetrievePermFullImageTimer = sentryMetrics.getTimer(
+ static final Timer getRetrievePermFullImageTimer = sentryMetrics.getTimer(
MetricRegistry.name(PermImageRetriever.class, "retrieve-perm-full-image"));
// The size of privilege changes for each retrievePermFullImage
- public static final Histogram getPrivilegeChangesHistogram = sentryMetrics.getHistogram(
+ static final Histogram getPrivilegeChangesHistogram = sentryMetrics.getHistogram(
MetricRegistry.name(PermImageRetriever.class, "retrieve-perm-full-image",
"privilege-changes-size"));
// The size of role changes for each retrievePermFullImage call
- public static final Histogram getRoleChangesHistogram = sentryMetrics.getHistogram(
+ static final Histogram getRoleChangesHistogram = sentryMetrics.getHistogram(
MetricRegistry.name(PermImageRetriever.class, "retrieve-perm-full-image",
"role-changes-size"));
// Metrics for retrievePathFullImage in PathImageRetriever
// The time used for each retrievePathFullImage
- public static final Timer getRetrievePathFullImageTimer = sentryMetrics.getTimer(
+ static final Timer getRetrievePathFullImageTimer = sentryMetrics.getTimer(
MetricRegistry.name(PathImageRetriever.class, "retrieve-path-full-image"));
// The size of path changes for each retrievePathFullImage
- public static final Histogram getPathChangesHistogram = sentryMetrics.getHistogram(
+ static final Histogram getPathChangesHistogram = sentryMetrics.getHistogram(
MetricRegistry.name(PathImageRetriever.class, "retrieve-path-full-image",
"path-changes-size"));
+ // Timer for getting path changes deltas
+ static final Timer getDeltaPathChangesTimer = sentryMetrics.getTimer(
+ MetricRegistry.name(PathDeltaRetriever.class, "path", "delta", "time")
+ );
+
+ // Histogram for the number of path changes processed for deltas
+ static final Histogram getDeltaPathChangesHistogram = sentryMetrics.getHistogram(
+ MetricRegistry.name(PathDeltaRetriever.class, "path", "delta", "size"));
+
+
+ // Timer for getting permission changes deltas
+ static final Timer getDeltaPermChangesTimer = sentryMetrics.getTimer(
+ MetricRegistry.name(PathDeltaRetriever.class, "perm", "delta", "time")
+ );
+
+ // Histogram for the number of permissions changes processed for deltas
+ static final Histogram getDeltaPermChangesHistogram = sentryMetrics.getHistogram(
+ MetricRegistry.name(PathDeltaRetriever.class, "perm", "delta", "size"));
+
private SentryHdfsMetricsUtil() {
// Make constructor private to avoid instantiation
}