You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by ab...@apache.org on 2021/11/07 11:22:08 UTC
[druid] branch master updated: Add more metrics for Jetty server
thread pool usage (#11113)
This is an automated email from the ASF dual-hosted git repository.
abhishek pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 8e7e679 Add more metrics for Jetty server thread pool usage (#11113)
8e7e679 is described below
commit 8e7e679984e95465f87dd97dad9022aacd00af04
Author: Jian Wang <wj...@gmail.com>
AuthorDate: Sun Nov 7 03:21:44 2021 -0800
Add more metrics for Jetty server thread pool usage (#11113)
Add more metrics for jetty server thread pool usage so we know if we have allocated enough http threads to handle requests.
---
docs/design/extensions-contrib/dropwizard.md | 32 +++++++++
docs/operations/metrics.md | 7 ++
.../src/main/resources/defaultMetrics.json | 7 ++
.../main/resources/defaultMetricDimensions.json | 11 ++-
.../initialization/jetty/JettyServerModule.java | 18 +++++
.../jetty/JettyServerModuleTest.java | 82 ++++++++++++++++++++++
website/.spelling | 5 ++
7 files changed, 161 insertions(+), 1 deletion(-)
diff --git a/docs/design/extensions-contrib/dropwizard.md b/docs/design/extensions-contrib/dropwizard.md
index c76afca..a2a8c34 100644
--- a/docs/design/extensions-contrib/dropwizard.md
+++ b/docs/design/extensions-contrib/dropwizard.md
@@ -629,6 +629,38 @@ Latest default metrics mapping can be found [here] (https://github.com/apache/dr
"priority"
],
"type": "gauge"
+ },
+ "jetty/numOpenConnections": {
+ "dimensions": [],
+ "type": "gauge"
+ },
+ "jetty/threadPool/total": {
+ "dimensions": [],
+ "type": "gauge"
+ },
+ "jetty/threadPool/idle": {
+ "dimensions": [],
+ "type": "gauge"
+ },
+ "jetty/threadPool/busy": {
+ "dimensions": [],
+ "type": "gauge"
+ },
+ "jetty/threadPool/isLowOnThreads": {
+ "dimensions": [],
+ "type": "gauge"
+ },
+ "jetty/threadPool/min": {
+ "dimensions": [],
+ "type": "gauge"
+ },
+ "jetty/threadPool/max": {
+ "dimensions": [],
+ "type": "gauge"
+ },
+ "jetty/threadPool/queueSize": {
+ "dimensions": [],
+ "type": "gauge"
}
}
```
diff --git a/docs/operations/metrics.md b/docs/operations/metrics.md
index e987c1c..1c7c736 100644
--- a/docs/operations/metrics.md
+++ b/docs/operations/metrics.md
@@ -97,6 +97,13 @@ Available Metrics
|Metric|Description|Normal Value|
|------|-----------|------------|
|`jetty/numOpenConnections`|Number of open jetty connections.|Not much higher than number of jetty threads.|
+|`jetty/threadPool/total`|Number of total workable threads allocated.|The number should equal to threadPoolNumIdleThreads + threadPoolNumBusyThreads.|
+|`jetty/threadPool/idle`|Number of idle threads.|Less than or equal to threadPoolNumTotalThreads. Non zero number means there is less work to do than configured capacity.|
+|`jetty/threadPool/busy`|Number of busy threads that has work to do from the worker queue.|Less than or equal to threadPoolNumTotalThreads.|
+|`jetty/threadPool/isLowOnThreads`|A rough indicator of whether number of total workable threads allocated is enough to handle the works in the work queue.|0|
+|`jetty/threadPool/min`|Number of minimum threads allocatable.|druid.server.http.numThreads plus a small fixed number of threads allocated for Jetty acceptors and selectors.|
+|`jetty/threadPool/max`|Number of maximum threads allocatable.|druid.server.http.numThreads plus a small fixed number of threads allocated for Jetty acceptors and selectors.|
+|`jetty/threadPool/queueSize`|Size of the worker queue.|Not much higher than druid.server.http.queueSize|
### Cache
diff --git a/extensions-contrib/opentsdb-emitter/src/main/resources/defaultMetrics.json b/extensions-contrib/opentsdb-emitter/src/main/resources/defaultMetrics.json
index c2d3845..ae01473 100644
--- a/extensions-contrib/opentsdb-emitter/src/main/resources/defaultMetrics.json
+++ b/extensions-contrib/opentsdb-emitter/src/main/resources/defaultMetrics.json
@@ -28,6 +28,13 @@
"type"
],
"jetty/numOpenConnections": [],
+ "jetty/threadPool/total": [],
+ "jetty/threadPool/idle": [],
+ "jetty/threadPool/busy": [],
+ "jetty/threadPool/isLowOnThreads": [],
+ "jetty/threadPool/min": [],
+ "jetty/threadPool/max": [],
+ "jetty/threadPool/queueSize": [],
"query/cache/delta/numEntries": [],
"query/cache/delta/sizeBytes": [],
"query/cache/delta/hits": [],
diff --git a/extensions-contrib/statsd-emitter/src/main/resources/defaultMetricDimensions.json b/extensions-contrib/statsd-emitter/src/main/resources/defaultMetricDimensions.json
index 298b440..5ac0886 100644
--- a/extensions-contrib/statsd-emitter/src/main/resources/defaultMetricDimensions.json
+++ b/extensions-contrib/statsd-emitter/src/main/resources/defaultMetricDimensions.json
@@ -130,5 +130,14 @@
"sys/cpu" : { "dimensions" : ["cpuName", "cpuTime"], "type" : "gauge"},
"coordinator-segment/count" : { "dimensions" : ["dataSource"], "type" : "gauge" },
- "historical-segment/count" : { "dimensions" : ["dataSource", "tier", "priority"], "type" : "gauge" }
+ "historical-segment/count" : { "dimensions" : ["dataSource", "tier", "priority"], "type" : "gauge" },
+
+ "jetty/numOpenConnections": { "dimensions" : [], "type" : "gauge" },
+ "jetty/threadPool/total": { "dimensions" : [], "type" : "gauge" },
+ "jetty/threadPool/idle": { "dimensions" : [], "type" : "gauge" },
+ "jetty/threadPool/busy": { "dimensions" : [], "type" : "gauge" },
+ "jetty/threadPool/isLowOnThreads": { "dimensions" : [], "type" : "gauge" },
+ "jetty/threadPool/min": { "dimensions" : [], "type" : "gauge" },
+ "jetty/threadPool/max": { "dimensions" : [], "type" : "gauge" },
+ "jetty/threadPool/queueSize": { "dimensions" : [], "type" : "gauge" }
}
diff --git a/server/src/main/java/org/apache/druid/server/initialization/jetty/JettyServerModule.java b/server/src/main/java/org/apache/druid/server/initialization/jetty/JettyServerModule.java
index 41a00a2..b722fb2 100644
--- a/server/src/main/java/org/apache/druid/server/initialization/jetty/JettyServerModule.java
+++ b/server/src/main/java/org/apache/druid/server/initialization/jetty/JettyServerModule.java
@@ -109,6 +109,7 @@ public class JettyServerModule extends JerseyServletModule
private static final AtomicInteger ACTIVE_CONNECTIONS = new AtomicInteger();
private static final String HTTP_1_1_STRING = "HTTP/1.1";
+ private static QueuedThreadPool jettyServerThreadPool = null;
@Override
protected void configureServlets()
@@ -229,6 +230,7 @@ public class JettyServerModule extends JerseyServletModule
}
threadPool.setDaemon(true);
+ jettyServerThreadPool = threadPool;
final Server server = new Server(threadPool);
@@ -528,6 +530,16 @@ public class JettyServerModule extends JerseyServletModule
final ServiceMetricEvent.Builder builder = new ServiceMetricEvent.Builder();
MonitorUtils.addDimensionsToBuilder(builder, dimensions);
emitter.emit(builder.build("jetty/numOpenConnections", ACTIVE_CONNECTIONS.get()));
+ if (jettyServerThreadPool != null) {
+ emitter.emit(builder.build("jetty/threadPool/total", jettyServerThreadPool.getThreads()));
+ emitter.emit(builder.build("jetty/threadPool/idle", jettyServerThreadPool.getIdleThreads()));
+ emitter.emit(builder.build("jetty/threadPool/isLowOnThreads", jettyServerThreadPool.isLowOnThreads() ? 1 : 0));
+ emitter.emit(builder.build("jetty/threadPool/min", jettyServerThreadPool.getMinThreads()));
+ emitter.emit(builder.build("jetty/threadPool/max", jettyServerThreadPool.getMaxThreads()));
+ emitter.emit(builder.build("jetty/threadPool/queueSize", jettyServerThreadPool.getQueueSize()));
+ emitter.emit(builder.build("jetty/threadPool/busy", jettyServerThreadPool.getBusyThreads()));
+ }
+
return true;
}
}
@@ -578,4 +590,10 @@ public class JettyServerModule extends JerseyServletModule
{
return ACTIVE_CONNECTIONS.get();
}
+
+ @VisibleForTesting
+ public static void setJettyServerThreadPool(QueuedThreadPool threadPool)
+ {
+ jettyServerThreadPool = threadPool;
+ }
}
diff --git a/server/src/test/java/org/apache/druid/server/initialization/jetty/JettyServerModuleTest.java b/server/src/test/java/org/apache/druid/server/initialization/jetty/JettyServerModuleTest.java
new file mode 100644
index 0000000..e4158d7
--- /dev/null
+++ b/server/src/test/java/org/apache/druid/server/initialization/jetty/JettyServerModuleTest.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.server.initialization.jetty;
+
+import org.apache.druid.java.util.common.Pair;
+import org.apache.druid.java.util.emitter.core.Emitter;
+import org.apache.druid.java.util.emitter.core.Event;
+import org.apache.druid.java.util.emitter.service.ServiceEmitter;
+import org.apache.druid.java.util.emitter.service.ServiceMetricEvent;
+import org.eclipse.jetty.util.thread.QueuedThreadPool;
+import org.junit.Assert;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+public class JettyServerModuleTest
+{
+ @Test
+ public void testJettyServerModule()
+ {
+ List<Event> events = new ArrayList<>();
+ ServiceEmitter serviceEmitter = new ServiceEmitter("service", "host", Mockito.mock(Emitter.class))
+ {
+ @Override
+ public void emit(Event event)
+ {
+ events.add(event);
+ }
+ };
+ QueuedThreadPool jettyServerThreadPool = Mockito.mock(QueuedThreadPool.class);
+ JettyServerModule.setJettyServerThreadPool(jettyServerThreadPool);
+ Mockito.when(jettyServerThreadPool.getThreads()).thenReturn(100);
+ Mockito.when(jettyServerThreadPool.getIdleThreads()).thenReturn(40);
+ Mockito.when(jettyServerThreadPool.isLowOnThreads()).thenReturn(true);
+ Mockito.when(jettyServerThreadPool.getMinThreads()).thenReturn(30);
+ Mockito.when(jettyServerThreadPool.getMaxThreads()).thenReturn(100);
+ Mockito.when(jettyServerThreadPool.getQueueSize()).thenReturn(50);
+ Mockito.when(jettyServerThreadPool.getBusyThreads()).thenReturn(60);
+
+ JettyServerModule.JettyMonitor jettyMonitor = new JettyServerModule.JettyMonitor("ds", "t0");
+ jettyMonitor.doMonitor(serviceEmitter);
+
+ Assert.assertEquals(8, events.size());
+ List<Pair<String, Number>> expectedEvents = Arrays.asList(
+ new Pair<>("jetty/numOpenConnections", 0),
+ new Pair<>("jetty/threadPool/total", 100),
+ new Pair<>("jetty/threadPool/idle", 40),
+ new Pair<>("jetty/threadPool/isLowOnThreads", 1),
+ new Pair<>("jetty/threadPool/min", 30),
+ new Pair<>("jetty/threadPool/max", 100),
+ new Pair<>("jetty/threadPool/queueSize", 50),
+ new Pair<>("jetty/threadPool/busy", 60)
+ );
+
+ for (int i = 0; i < expectedEvents.size(); i++) {
+ Pair<String, Number> expected = expectedEvents.get(i);
+ ServiceMetricEvent actual = (ServiceMetricEvent) (events.get(i));
+ Assert.assertEquals(expected.lhs, actual.getMetric());
+ Assert.assertEquals(expected.rhs, actual.getValue());
+ }
+ }
+}
diff --git a/website/.spelling b/website/.spelling
index f132423..c2b9dd9 100644
--- a/website/.spelling
+++ b/website/.spelling
@@ -1293,6 +1293,8 @@ bufferpoolName
cms
cpuName
cpuTime
+druid.server.http.numThreads
+druid.server.http.queueSize
fsDevName
fsDirName
fsOptions
@@ -1318,6 +1320,9 @@ remoteAddress
serviceName
taskStatus
taskType
+threadPoolNumBusyThreads.
+threadPoolNumIdleThreads
+threadPoolNumTotalThreads.
- ../docs/operations/other-hadoop.md
CDH
Classloader
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org