You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sentry.apache.org by ak...@apache.org on 2017/06/28 00:57:04 UTC
sentry git commit: SENTRY-1820 Add JSON file reporter for Sentry
metrics
Repository: sentry
Updated Branches:
refs/heads/SENTRY-1820 [created] 010446fc7
SENTRY-1820 Add JSON file reporter for Sentry metrics
Project: http://git-wip-us.apache.org/repos/asf/sentry/repo
Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/010446fc
Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/010446fc
Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/010446fc
Branch: refs/heads/SENTRY-1820
Commit: 010446fc7d71a7e0f5ee65379a2565a9a355d2e2
Parents: 8d53da5
Author: Alexander Kolbasov <ak...@cloudera.com>
Authored: Tue Jun 27 17:53:00 2017 -0700
Committer: Alexander Kolbasov <ak...@cloudera.com>
Committed: Tue Jun 27 17:53:00 2017 -0700
----------------------------------------------------------------------
.../sentry/service/thrift/ServiceConstants.java | 5 +-
.../db/service/thrift/SentryMetrics.java | 106 +++++++++++++++++--
2 files changed, 101 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sentry/blob/010446fc/sentry-service/sentry-service-common/src/main/java/org/apache/sentry/service/thrift/ServiceConstants.java
----------------------------------------------------------------------
diff --git a/sentry-service/sentry-service-common/src/main/java/org/apache/sentry/service/thrift/ServiceConstants.java b/sentry-service/sentry-service-common/src/main/java/org/apache/sentry/service/thrift/ServiceConstants.java
index de97a31..affb8a4 100644
--- a/sentry-service/sentry-service-common/src/main/java/org/apache/sentry/service/thrift/ServiceConstants.java
+++ b/sentry-service/sentry-service-common/src/main/java/org/apache/sentry/service/thrift/ServiceConstants.java
@@ -166,11 +166,14 @@ public class ServiceConstants {
public static final Boolean SENTRY_WEB_ENABLE_DEFAULT = false;
public static final String SENTRY_WEB_PORT = "sentry.service.web.port";
public static final int SENTRY_WEB_PORT_DEFAULT = 29000;
- // Reporter is either "console" or "jmx"
+ // Reporter is either "console", "log" or "jmx"
public static final String SENTRY_REPORTER = "sentry.service.reporter";
// for console reporter, reporting interval in seconds
public static final String SENTRY_REPORTER_INTERVAL_SEC =
"sentry.service.reporter.interval.sec";
+ public static final String SENTRY_JSON_REPORTER_FILE = "sentry.service.reporter.file";
+ public static final String SENTRY_JSON_REPORTER_FILE_DEFAULT = "/tmp/sentry-metrics.json";
+
// Report every 5 minutes by default
public static final int SENTRY_REPORTER_INTERVAL_DEFAULT = 300;
http://git-wip-us.apache.org/repos/asf/sentry/blob/010446fc/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryMetrics.java
----------------------------------------------------------------------
diff --git a/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryMetrics.java b/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryMetrics.java
index af70c99..1a5fd24 100644
--- a/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryMetrics.java
+++ b/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryMetrics.java
@@ -21,20 +21,35 @@ import com.codahale.metrics.*;
import static com.codahale.metrics.MetricRegistry.name;
+import com.codahale.metrics.json.MetricsModule;
import com.codahale.metrics.jvm.BufferPoolMetricSet;
import com.codahale.metrics.jvm.GarbageCollectorMetricSet;
import com.codahale.metrics.jvm.MemoryUsageGaugeSet;
import com.codahale.metrics.jvm.ThreadStatesGaugeSet;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalFileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.sentry.provider.db.service.persistent.SentryStore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static java.io.File.createTempFile;
import static org.apache.sentry.provider.db.service.thrift.SentryMetricsServletContextListener.METRIC_REGISTRY;
import static org.apache.sentry.service.thrift.ServiceConstants.ServerConfig;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
import java.lang.management.ManagementFactory;
import java.util.Map;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -42,6 +57,13 @@ import java.util.concurrent.atomic.AtomicBoolean;
* A singleton class which holds metrics related utility functions as well as the list of metrics
*/
public final class SentryMetrics {
+ public enum Reporting {
+ JMX,
+ CONSOLE,
+ LOG,
+ JSON,
+ }
+
private static final Logger LOGGER = LoggerFactory
.getLogger(SentryMetrics.class);
@@ -152,8 +174,8 @@ public final class SentryMetrics {
switch(SentryMetrics.Reporting.valueOf(reporter.toUpperCase())) {
case CONSOLE:
- LOGGER.info(String.format("Enabled console metrics reporter with %d seconds interval",
- reportInterval));
+ LOGGER.info("Enabled console metrics reporter with {} seconds interval",
+ reportInterval);
final ConsoleReporter consoleReporter =
ConsoleReporter.forRegistry(METRIC_REGISTRY)
.convertRatesTo(TimeUnit.SECONDS)
@@ -170,8 +192,8 @@ public final class SentryMetrics {
jmxReporter.start();
break;
case LOG:
- LOGGER.info(String.format("Enabled Log4J metrics reporter with %d seconds interval",
- reportInterval));
+ LOGGER.info("Enabled Log4J metrics reporter with {} seconds interval",
+ reportInterval);
final Slf4jReporter logReporter = Slf4jReporter.forRegistry(METRIC_REGISTRY)
.outputTo(LOGGER)
.convertRatesTo(TimeUnit.SECONDS)
@@ -179,8 +201,13 @@ public final class SentryMetrics {
.build();
logReporter.start(reportInterval, TimeUnit.SECONDS);
break;
+ case JSON:
+ LOGGER.info("Enabled JSON metrics reporter with {} seconds interval", reportInterval);
+ final JsonFileReporter jsonReporter = new JsonFileReporter(conf, reportInterval, TimeUnit.SECONDS);
+ jsonReporter.start();
+ break;
default:
- LOGGER.warn("Invalid metrics reporter " + reporter);
+ LOGGER.warn("Invalid metrics reporter {}", reporter);
break;
}
}
@@ -200,9 +227,70 @@ public final class SentryMetrics {
}
}
- public enum Reporting {
- JMX,
- CONSOLE,
- LOG,
+ static class JsonFileReporter implements AutoCloseable, Runnable {
+ private static final short PERMISSIONS = 0644;
+
+ private ScheduledExecutorService executor = null;
+ private final ObjectMapper jsonMapper =
+ new ObjectMapper().registerModule(new MetricsModule(TimeUnit.SECONDS,
+ TimeUnit.MILLISECONDS,
+ false));
+ private final Configuration conf;
+ private final String pathString;
+ private final long interval;
+ private final TimeUnit unit;
+
+ JsonFileReporter(Configuration conf, long interval, TimeUnit unit) {
+ this.conf = conf;
+ pathString = conf.get(ServerConfig.SENTRY_JSON_REPORTER_FILE,
+ ServerConfig.SENTRY_JSON_REPORTER_FILE_DEFAULT);
+ this.interval = interval;
+ this.unit = unit;
+ }
+
+ private void start() {
+ executor = Executors.newScheduledThreadPool(1,
+ new ThreadFactoryBuilder().setNameFormat("json-reporter").build());
+ executor.scheduleAtFixedRate(this, 0, interval, unit);
+ }
+
+ @Override
+ public void run() {
+ String json = null;
+ try {
+ json = jsonMapper.writerWithDefaultPrettyPrinter().writeValueAsString(METRIC_REGISTRY);
+ } catch (JsonProcessingException e) {
+ LOGGER.error("Error converting metrics to JSON", e);
+ return;
+ }
+ File tmpFile = null;
+ try {
+ tmpFile = createTempFile("sentry-json", null);
+ } catch (IOException e) {
+ LOGGER.error("failed to create temp file for JSON metrics", e);
+ }
+
+ assert tmpFile != null;
+ try (LocalFileSystem fs = FileSystem.getLocal(conf);
+ BufferedWriter bw = new BufferedWriter(new FileWriter(tmpFile))) {
+ bw.write(json);
+ Path tmpPath = new Path(tmpFile.getAbsolutePath());
+ fs.setPermission(tmpPath, FsPermission.createImmutable(PERMISSIONS));
+ Path path = new Path(pathString);
+ fs.rename(tmpPath, path);
+ fs.setPermission(path, FsPermission.createImmutable(PERMISSIONS));
+ } catch (IOException e) {
+ LOGGER.warn("Error writing JSON metrics", e);
+ }
+ }
+
+ @Override
+ public void close() {
+ if (executor != null) {
+ executor.shutdownNow();
+ executor = null;
+ }
+ }
+
}
}