You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by ja...@apache.org on 2014/06/11 05:52:19 UTC
[34/61] [abbrv] git commit: add servlets to expose server level
metrics.
add servlets to expose server level metrics.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/4eea03a0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/4eea03a0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/4eea03a0
Branch: refs/heads/master
Commit: 4eea03a052d1b0a9190b9d1512088da9f81cc037
Parents: ed260f1
Author: Jacques Nadeau <ja...@apache.org>
Authored: Sat Jun 7 08:38:55 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Sun Jun 8 19:13:06 2014 -0700
----------------------------------------------------------------------
exec/java-exec/pom.xml | 5 ++
.../apache/drill/exec/metrics/DrillMetrics.java | 84 +++++++++++++++-----
.../drill/exec/server/BootStrapContext.java | 3 +-
.../org/apache/drill/exec/server/Drillbit.java | 7 ++
.../org/apache/drill/exec/work/WorkManager.java | 16 ++++
.../java/org/apache/drill/exec/ExecTest.java | 6 ++
6 files changed, 98 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/4eea03a0/exec/java-exec/pom.xml
----------------------------------------------------------------------
diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml
index da0c96a..3c245b8 100644
--- a/exec/java-exec/pom.xml
+++ b/exec/java-exec/pom.xml
@@ -152,6 +152,11 @@
<version>3.0.1</version>
</dependency>
<dependency>
+ <groupId>com.codahale.metrics</groupId>
+ <artifactId>metrics-servlets</artifactId>
+ <version>3.0.1</version>
+ </dependency>
+ <dependency>
<groupId>javax.inject</groupId>
<artifactId>javax.inject</artifactId>
<version>1</version>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/4eea03a0/exec/java-exec/src/main/java/org/apache/drill/exec/metrics/DrillMetrics.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/metrics/DrillMetrics.java b/exec/java-exec/src/main/java/org/apache/drill/exec/metrics/DrillMetrics.java
index eef1181..d3bc93e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/metrics/DrillMetrics.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/metrics/DrillMetrics.java
@@ -17,25 +17,47 @@
*/
package org.apache.drill.exec.metrics;
-import com.codahale.metrics.JmxReporter;
-import com.codahale.metrics.MetricRegistry;
-import com.codahale.metrics.Slf4jReporter;
+import java.lang.management.ManagementFactory;
+import java.util.Map.Entry;
+import java.util.concurrent.TimeUnit;
+
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.exec.ExecConstants;
-import org.apache.drill.exec.client.DrillClient;
-import java.util.concurrent.TimeUnit;
+import com.codahale.metrics.JmxReporter;
+import com.codahale.metrics.Metric;
+import com.codahale.metrics.MetricFilter;
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.MetricSet;
+import com.codahale.metrics.Slf4jReporter;
+import com.codahale.metrics.jvm.BufferPoolMetricSet;
+import com.codahale.metrics.jvm.GarbageCollectorMetricSet;
+import com.codahale.metrics.jvm.MemoryUsageGaugeSet;
+import com.codahale.metrics.jvm.ThreadStatesGaugeSet;
public class DrillMetrics {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillMetrics.class);
static final DrillConfig config = DrillConfig.create();
- private DrillMetrics() {}
+ private DrillMetrics() {
+ }
private static class RegistryHolder {
- public static final MetricRegistry REGISTRY = new MetricRegistry();
- private static JmxReporter jmxReporter = getJmxReporter();
- private static Slf4jReporter logReporter = getLogReporter();
+ public static final MetricRegistry REGISTRY;
+// private static JmxReporter jmxReporter = getJmxReporter();
+// private static Slf4jReporter logReporter = getLogReporter();
+
+ static {
+ REGISTRY = new MetricRegistry();
+ registerSysStats();
+ }
+
+ private static void registerSysStats(){
+ REGISTRY.registerAll(new GarbageCollectorMetricSet());
+ REGISTRY.registerAll(new BufferPoolMetricSet(ManagementFactory.getPlatformMBeanServer()));
+ REGISTRY.registerAll(new MemoryUsageGaugeSet());
+ REGISTRY.registerAll(new ThreadStatesGaugeSet());
+ }
private static JmxReporter getJmxReporter() {
if (config.getBoolean(ExecConstants.METRICS_JMX_OUTPUT_ENABLED)) {
@@ -43,25 +65,43 @@ public class DrillMetrics {
reporter.start();
return reporter;
- } else return null;
+ } else
+ return null;
+ }
+
+ private static Slf4jReporter getLogReporter() {
+ if (config.getBoolean(ExecConstants.METRICS_LOG_OUTPUT_ENABLED)) {
+ Slf4jReporter reporter = Slf4jReporter.forRegistry(getInstance()).outputTo(logger)
+ .convertRatesTo(TimeUnit.SECONDS).convertDurationsTo(TimeUnit.MILLISECONDS).build();
+ reporter.start(config.getInt(ExecConstants.METRICS_LOG_OUTPUT_INTERVAL), TimeUnit.SECONDS);
+
+ return reporter;
+ } else
+ return null;
+ }
+ }
+
+ private static void registerAll(String prefix, MetricSet metricSet, MetricRegistry registry) {
+ for (Entry<String, Metric> entry : metricSet.getMetrics().entrySet()) {
+ if (entry.getValue() instanceof MetricSet) {
+ registerAll(prefix + "." + entry.getKey(), (MetricSet) entry.getValue(), registry);
+ } else {
+ registry.register(prefix + "." + entry.getKey(), entry.getValue());
+ }
}
- private static Slf4jReporter getLogReporter() {
- if(config.getBoolean(ExecConstants.METRICS_LOG_OUTPUT_ENABLED)) {
- Slf4jReporter reporter = Slf4jReporter.forRegistry(getInstance())
- .outputTo(logger)
- .convertRatesTo(TimeUnit.SECONDS)
- .convertDurationsTo(TimeUnit.MILLISECONDS)
- .build();
- reporter.start(config.getInt(ExecConstants.METRICS_LOG_OUTPUT_INTERVAL), TimeUnit.SECONDS);
-
- return reporter;
- } else return null;
- }
}
public static MetricRegistry getInstance() {
return RegistryHolder.REGISTRY;
}
+ public static void resetMetrics(){
+ RegistryHolder.REGISTRY.removeMatching(new MetricFilter(){
+ @Override
+ public boolean matches(String name, Metric metric) {
+ return true;
+ }});
+ RegistryHolder.registerSysStats();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/4eea03a0/exec/java-exec/src/main/java/org/apache/drill/exec/server/BootStrapContext.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/BootStrapContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/BootStrapContext.java
index 4261885..ddfb42c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/BootStrapContext.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/BootStrapContext.java
@@ -25,6 +25,7 @@ import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.memory.TopLevelAllocator;
+import org.apache.drill.exec.metrics.DrillMetrics;
import org.apache.drill.exec.rpc.NamedThreadFactory;
import com.codahale.metrics.MetricRegistry;
@@ -43,7 +44,7 @@ public class BootStrapContext implements Closeable{
this.config = config;
this.loop = new NioEventLoopGroup(config.getInt(ExecConstants.BIT_SERVER_RPC_THREADS), new NamedThreadFactory("BitServer-"));
this.loop2 = new NioEventLoopGroup(config.getInt(ExecConstants.BIT_SERVER_RPC_THREADS), new NamedThreadFactory("BitClient-"));
- this.metrics = new MetricRegistry();
+ this.metrics = DrillMetrics.getInstance();
this.allocator = new TopLevelAllocator(config);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/4eea03a0/exec/java-exec/src/main/java/org/apache/drill/exec/server/Drillbit.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/Drillbit.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/Drillbit.java
index f6637cf..43cc2b9 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/Drillbit.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/Drillbit.java
@@ -19,6 +19,8 @@ package org.apache.drill.exec.server;
import java.io.Closeable;
+import javax.servlet.Servlet;
+
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.cache.DistributedCache;
@@ -41,6 +43,8 @@ import org.eclipse.jetty.servlet.ServletContextHandler;
import org.eclipse.jetty.servlet.ServletHolder;
import org.glassfish.jersey.servlet.ServletContainer;
+import com.codahale.metrics.servlets.MetricsServlet;
+import com.codahale.metrics.servlets.ThreadDumpServlet;
import com.google.common.io.Closeables;
/**
@@ -129,6 +133,9 @@ public class Drillbit implements Closeable{
// h.setInitParameter(ServerProperties.PROVIDER_PACKAGES, "org.apache.drill.exec.server");
h.setInitOrder(1);
context.addServlet(h, "/*");
+ context.addServlet(new ServletHolder(new MetricsServlet(this.context.getMetrics())), "/status/metrics");
+ context.addServlet(new ServletHolder(new ThreadDumpServlet()), "/status/threads");
+
embeddedJetty.start();
System.out.println("");
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/4eea03a0/exec/java-exec/src/main/java/org/apache/drill/exec/work/WorkManager.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/WorkManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/WorkManager.java
index c037a4c..bddc1cf 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/WorkManager.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/WorkManager.java
@@ -50,6 +50,8 @@ import org.apache.drill.exec.work.fragment.FragmentExecutor;
import org.apache.drill.exec.work.fragment.FragmentManager;
import org.apache.drill.exec.work.user.UserWorker;
+import com.codahale.metrics.Gauge;
+import com.codahale.metrics.MetricRegistry;
import com.google.common.collect.Maps;
import com.google.common.collect.Queues;
@@ -92,6 +94,20 @@ public class WorkManager implements Closeable{
// executor = Executors.newFixedThreadPool(dContext.getConfig().getInt(ExecConstants.EXECUTOR_THREADS)
executor = Executors.newCachedThreadPool(new NamedThreadFactory("WorkManager-"));
eventThread.start();
+ dContext.getMetrics().register(MetricRegistry.name("drill.exec.work.running_fragments." + dContext.getEndpoint().getUserPort()),
+ new Gauge<Integer>() {
+ @Override
+ public Integer getValue() {
+ return runningFragments.size();
+ }
+ });
+ dContext.getMetrics().register(MetricRegistry.name("drill.exec.work.pendingTasks" + dContext.getEndpoint().getUserPort()),
+ new Gauge<Integer>() {
+ @Override
+ public Integer getValue() {
+ return pendingTasks.size();
+ }
+ });
}
public WorkEventBus getWorkBus(){
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/4eea03a0/exec/java-exec/src/test/java/org/apache/drill/exec/ExecTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/ExecTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/ExecTest.java
index 7be8cc5..e7c6dc0 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/ExecTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/ExecTest.java
@@ -17,9 +17,15 @@
*/
package org.apache.drill.exec;
+import org.apache.drill.exec.metrics.DrillMetrics;
import org.apache.drill.test.DrillTest;
+import org.junit.After;
public class ExecTest extends DrillTest{
+ @After
+ public void clear(){
+ DrillMetrics.resetMetrics();
+ }
}