You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sentry.apache.org by ak...@apache.org on 2017/07/28 17:06:27 UTC
[09/50] [abbrv] sentry git commit: SENTRY-1696: Expose time spent
creating the initial snapshot as a metric (Alex Kolbasov,
reviewed by Vamsee Yarlagadda)
SENTRY-1696: Expose time spent creating the initial snapshot as a metric (Alex Kolbasov, reviewed by Vamsee Yarlagadda)
Project: http://git-wip-us.apache.org/repos/asf/sentry/repo
Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/2fcd69aa
Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/2fcd69aa
Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/2fcd69aa
Branch: refs/heads/master
Commit: 2fcd69aa2ed6b0f80c5c0e91a6a84ce24459a336
Parents: 0370597
Author: Alexander Kolbasov <ak...@cloudera.com>
Authored: Wed Jul 12 19:45:52 2017 +0200
Committer: Alexander Kolbasov <ak...@cloudera.com>
Committed: Wed Jul 12 19:45:52 2017 +0200
----------------------------------------------------------------------
.../service/thrift/FullUpdateInitializer.java | 24 +++++++++++++++++++-
.../sentry/service/thrift/HMSFollower.java | 17 +++++++++++++-
2 files changed, 39 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sentry/blob/2fcd69aa/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/FullUpdateInitializer.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/FullUpdateInitializer.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/FullUpdateInitializer.java
index 1490581..b74fa50 100644
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/FullUpdateInitializer.java
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/FullUpdateInitializer.java
@@ -17,6 +17,7 @@
*/
package org.apache.sentry.service.thrift;
+import com.codahale.metrics.Counter;
import com.google.common.collect.ImmutableMap;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.metastore.api.Database;
@@ -25,6 +26,7 @@ import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.sentry.hdfs.PathsUpdate;
import org.apache.sentry.hdfs.SentryMalformedPathException;
import org.apache.sentry.hdfs.ServiceConstants.ServerConfig;
+import org.apache.sentry.provider.db.service.thrift.SentryMetrics;
import org.apache.thrift.TException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -46,6 +48,8 @@ import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
+import static com.codahale.metrics.MetricRegistry.name;
+
/**
* Manage fetching full snapshot from HMS.
* Snapshot is represented as a map from the hive object name to
@@ -103,6 +107,18 @@ public final class FullUpdateInitializer implements AutoCloseable {
new ObjectMapping(Collections.<String, Set<String>>emptyMap());
private final HiveConnectionFactory clientFactory;
+ /** Total number of database objects */
+ private final Counter databaseCount = SentryMetrics.getInstance()
+ .getCounter(name(FullUpdateInitializer.class, "total", "db"));
+
+ /** Total number of table objects */
+ private final Counter tableCount = SentryMetrics.getInstance()
+ .getCounter(name(FullUpdateInitializer.class, "total", "tables"));
+
+ /** Total number of partition objects */
+ private final Counter partitionCount = SentryMetrics.getInstance()
+ .getCounter(name(FullUpdateInitializer.class, "total", "partitions"));
+
/**
* Extract path (not starting with "/") from the full URI
* @param uri - resource URI (usually with scheme)
@@ -323,7 +339,10 @@ public final class FullUpdateInitializer implements AutoCloseable {
String tableName = safeIntern(tbl.getTableName().toLowerCase());
String authzObject = (dbName + "." + tableName).intern();
- List<String> tblPartNames = client.getClient().listPartitionNames(dbName, tableName, (short) -1);
+ List<String> tblPartNames =
+ client.getClient().listPartitionNames(dbName, tableName, (short) -1);
+ // Count total number of partitions
+ partitionCount.inc(tblPartNames.size());
for (int i = 0; i < tblPartNames.size(); i += maxPartitionsPerCall) {
List<String> partsToFetch = tblPartNames.subList(i,
Math.min(i + maxPartitionsPerCall, tblPartNames.size()));
@@ -359,6 +378,7 @@ public final class FullUpdateInitializer implements AutoCloseable {
DbTask(String dbName) {
//Database names are case insensitive
this.dbName = safeIntern(dbName.toLowerCase());
+ databaseCount.inc();
}
@Override
@@ -372,6 +392,8 @@ public final class FullUpdateInitializer implements AutoCloseable {
return emptyObjectMapping;
}
List<String> allTblStr = client.getClient().getAllTables(dbName);
+ // Count total number of tables
+ tableCount.inc(allTblStr.size());
for (int i = 0; i < allTblStr.size(); i += maxTablesPerCall) {
List<String> tablesToFetch = allTblStr.subList(i,
Math.min(i + maxTablesPerCall, allTblStr.size()));
http://git-wip-us.apache.org/repos/asf/sentry/blob/2fcd69aa/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java
index 2d581f7..1b6ae18 100644
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java
@@ -17,6 +17,9 @@
*/
package org.apache.sentry.service.thrift;
+import com.codahale.metrics.Counter;
+import com.codahale.metrics.Timer;
+import com.codahale.metrics.Timer.Context;
import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
@@ -35,6 +38,7 @@ import org.apache.sentry.hdfs.PermissionsUpdate;
import org.apache.sentry.hdfs.service.thrift.TPrivilegeChanges;
import org.apache.sentry.provider.db.SentryPolicyStorePlugin;
import org.apache.sentry.provider.db.service.persistent.SentryStore;
+import org.apache.sentry.provider.db.service.thrift.SentryMetrics;
import org.apache.sentry.provider.db.service.thrift.TSentryAuthorizable;
import org.apache.thrift.TException;
import org.slf4j.Logger;
@@ -52,6 +56,7 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutionException;
+import static com.codahale.metrics.MetricRegistry.name;
import static org.apache.sentry.binding.hive.conf.HiveAuthzConf.AuthzConfVars.AUTHZ_SYNC_CREATE_WITH_POLICY_STORE;
import static org.apache.sentry.binding.hive.conf.HiveAuthzConf.AuthzConfVars.AUTHZ_SYNC_DROP_WITH_POLICY_STORE;
import static org.apache.sentry.hdfs.Updateable.Update;
@@ -77,6 +82,14 @@ public class HMSFollower implements Runnable, AutoCloseable {
private boolean needLogHMSSupportReady = true;
private final LeaderStatusMonitor leaderMonitor;
+ private static final String SNAPSHOT = "snapshot";
+ /** Measures time to get full snapshot */
+ private final Timer updateTimer = SentryMetrics.getInstance()
+ .getTimer(name(FullUpdateInitializer.class, SNAPSHOT));
+ /** Number of times update failed */
+ private final Counter failedSnapshotsCount = SentryMetrics.getInstance()
+ .getCounter(name(FullUpdateInitializer.class, "failed"));
+
HMSFollower(Configuration conf, SentryStore store, LeaderStatusMonitor leaderMonitor,
HiveSimpleConnectionFactory hiveConnectionFactory) {
authzConf = conf;
@@ -299,11 +312,13 @@ public class HMSFollower implements Runnable, AutoCloseable {
throws TException, ExecutionException {
LOGGER.info("Request full HMS snapshot");
try (FullUpdateInitializer updateInitializer =
- new FullUpdateInitializer(hiveConnectionFactory, authzConf)) {
+ new FullUpdateInitializer(hiveConnectionFactory, authzConf);
+ Context context = updateTimer.time()) {
Map<String, Set<String>> pathsUpdate = updateInitializer.getFullHMSSnapshot();
LOGGER.info("Obtained full HMS snapshot");
return pathsUpdate;
} catch (Exception ignored) {
+ failedSnapshotsCount.inc();
// Caller will retry later
return Collections.emptyMap();
}