You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2021/01/10 12:43:29 UTC
[incubator-doris] branch master updated: [Enhancement] Add more
comprehensive prometheus jvm thread metrics on fe (#5112)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new f2a11fe [Enhancement] Add more comprehensive prometheus jvm thread metrics on fe (#5112)
f2a11fe is described below
commit f2a11fe1f7c4f6cdab5d5361d6d41d063b6fd3d3
Author: Dam1029 <99...@qq.com>
AuthorDate: Sun Jan 10 20:43:17 2021 +0800
[Enhancement] Add more comprehensive prometheus jvm thread metrics on fe (#5112)
Currently, fe thread metrics is very simple, only have thread count and peak_count.
I think we may need more comprehensive prometheus jvm thread metrics on fe.
This will be useful when we want to analysis fe's running status.
---
.../operation/monitor-metrics/fe-metrics.md | 48 +++++++++++++++
.../operation/monitor-metrics/fe-metrics.md | 48 +++++++++++++++
.../org/apache/doris/common/proc/JvmProcDir.java | 6 ++
.../doris/metric/PrometheusMetricVisitor.java | 6 ++
.../org/apache/doris/monitor/jvm/JvmStats.java | 71 +++++++++++++++++++++-
5 files changed, 176 insertions(+), 3 deletions(-)
diff --git a/docs/en/administrator-guide/operation/monitor-metrics/fe-metrics.md b/docs/en/administrator-guide/operation/monitor-metrics/fe-metrics.md
index 7a2afae..4c4e3f7 100644
--- a/docs/en/administrator-guide/operation/monitor-metrics/fe-metrics.md
+++ b/docs/en/administrator-guide/operation/monitor-metrics/fe-metrics.md
@@ -75,3 +75,51 @@ Value of the `Tcp: OutSegs` field in `/proc/net/snmp`. Represents the number of
Use `(NEW_tcp_retrans_segs - OLD_tcp_retrans_segs) / (NEW_tcp_out_segs - OLD_tcp_out_segs)` can calculate the retrans rate of TCP packets.
Usually used to troubleshoot network problems.
+
+### `jvm_thread{type="count"}`
+
+Value of the `count` type in `jvm_thread`. Represents the current number of live threads including both daemon and non-daemon threads.
+
+Usually used to troubleshoot jvm threads problems for FE.
+
+### `jvm_thread{type="peak_count"}`
+
+Value of the `peak_count` type in `jvm_thread`. Represents the current number of live threads including both daemon and non-daemon threads.
+
+Usually used to troubleshoot jvm threads problems for FE.
+
+### `jvm_thread{type="new_count"}`
+
+Value of the `new_count` type in `jvm_thread`. Represents the current number of threads which state is NEW.
+
+Usually used to troubleshoot jvm threads problems for FE.
+
+### `jvm_thread{type="runnable_count"}`
+
+Value of the `runnable_count` type in `jvm_thread`. Represents the current number of threads which state is RUNNABLE.
+
+Usually used to troubleshoot jvm threads problems for FE.
+
+### `jvm_thread{type="blocked_count"}`
+
+Value of the `blocked_count` type in `jvm_thread`. Represents the current number of threads which state is BLOCKED.
+
+Usually used to troubleshoot jvm threads problems for FE.
+
+### `jvm_thread{type="waiting_count"}`
+
+Value of the `waiting_count` type in `jvm_thread`. Represents the current number of threads which state is WAITING.
+
+Usually used to troubleshoot jvm threads problems for FE.
+
+### `jvm_thread{type="timed_waiting_count"}`
+
+Value of the `timed_waiting_count` type in `jvm_thread`. Represents the current number of threads which state is TIMED_WAITING.
+
+Usually used to troubleshoot jvm threads problems for FE.
+
+### `jvm_thread{type="terminated_count"}`
+
+Value of the `terminated_count` type in `jvm_thread`. Represents the current number of threads which state is TERMINATED.
+
+Usually used to troubleshoot jvm threads problems for FE.
\ No newline at end of file
diff --git a/docs/zh-CN/administrator-guide/operation/monitor-metrics/fe-metrics.md b/docs/zh-CN/administrator-guide/operation/monitor-metrics/fe-metrics.md
index aaa1854..8b58e9f 100644
--- a/docs/zh-CN/administrator-guide/operation/monitor-metrics/fe-metrics.md
+++ b/docs/zh-CN/administrator-guide/operation/monitor-metrics/fe-metrics.md
@@ -75,3 +75,51 @@ FE 的监控项可以通过以下方式访问：
通过 `(NEW_tcp_tcp_retrans_segs - OLD_tcp_retrans_segs) / (NEW_tcp_out_segs - OLD_tcp_out_segs)` 可以计算 TCP 重传率。
通常用于排查网络问题。
+
+### `jvm_thread{type="count"}`
+
+该监控项表示FE节点当前JVM总的线程数量，包含daemon线程和非daemon线程。
+
+通常用于排查FE节点的JVM线程运行问题。
+
+### `jvm_thread{type="peak_count"}`
+
+该监控项表示FE节点从JVM启动以来的最大峰值线程数量。
+
+通常用于排查FE节点的JVM线程运行问题。
+
+### `jvm_thread{type="new_count"}`
+
+该监控项表示FE节点JVM中处于NEW状态的线程数量。
+
+通常用于排查FE节点的JVM线程运行问题。
+
+### `jvm_thread{type="runnable_count"}`
+
+该监控项表示FE节点JVM中处于RUNNABLE状态的线程数量。
+
+通常用于排查FE节点的JVM线程运行问题。
+
+### `jvm_thread{type="blocked_count"}`
+
+该监控项表示FE节点JVM中处于BLOCKED状态的线程数量。
+
+通常用于排查FE节点的JVM线程运行问题。
+
+### `jvm_thread{type="waiting_count"}`
+
+该监控项表示FE节点JVM中处于WAITING状态的线程数量。
+
+通常用于排查FE节点的JVM线程运行问题。
+
+### `jvm_thread{type="timed_waiting_count"}`
+
+该监控项表示FE节点JVM中处于TIMED_WAITING状态的线程数量。
+
+通常用于排查FE节点的JVM线程运行问题。
+
+### `jvm_thread{type="terminated_count"}`
+
+该监控项表示FE节点JVM中处于TERMINATED状态的线程数量。
+
+通常用于排查FE节点的JVM线程运行问题。
\ No newline at end of file
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/JvmProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/JvmProcDir.java
index ca3cffa..fce113f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/JvmProcDir.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/JvmProcDir.java
@@ -94,6 +94,12 @@ public class JvmProcDir implements ProcNodeInterface {
Threads threads = jvmStats.getThreads();
result.addRow(genRow("threads count", threads.getCount()));
result.addRow(genRow("threads peak count", threads.getPeakCount()));
+ result.addRow(genRow("threads new count", threads.getThreadsNewCount()));
+ result.addRow(genRow("threads runnable count", threads.getThreadsRunnableCount()));
+ result.addRow(genRow("threads blocked count", threads.getThreadsBlockedCount()));
+ result.addRow(genRow("threads waiting count", threads.getThreadsWaitingCount()));
+ result.addRow(genRow("threads timed_waiting count", threads.getThreadsTimedWaitingCount()));
+ result.addRow(genRow("threads terminated count", threads.getThreadsTerminatedCount()));
return result;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/metric/PrometheusMetricVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/metric/PrometheusMetricVisitor.java
index 6deba5b..c692ebc 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/metric/PrometheusMetricVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/metric/PrometheusMetricVisitor.java
@@ -134,6 +134,12 @@ public class PrometheusMetricVisitor extends MetricVisitor {
sb.append(Joiner.on(" ").join(TYPE, JVM_THREAD, "gauge\n"));
sb.append(JVM_THREAD).append("{type=\"count\"} ").append(threads.getCount()).append("\n");
sb.append(JVM_THREAD).append("{type=\"peak_count\"} ").append(threads.getPeakCount()).append("\n");
+ sb.append(JVM_THREAD).append("{type=\"new_count\"} ").append(threads.getThreadsNewCount()).append("\n");
+ sb.append(JVM_THREAD).append("{type=\"runnable_count\"} ").append(threads.getThreadsRunnableCount()).append("\n");
+ sb.append(JVM_THREAD).append("{type=\"blocked_count\"} ").append(threads.getThreadsBlockedCount()).append("\n");
+ sb.append(JVM_THREAD).append("{type=\"waiting_count\"} ").append(threads.getThreadsWaitingCount()).append("\n");
+ sb.append(JVM_THREAD).append("{type=\"timed_waiting_count\"} ").append(threads.getThreadsTimedWaitingCount()).append("\n");
+ sb.append(JVM_THREAD).append("{type=\"terminated_count\"} ").append(threads.getThreadsTerminatedCount()).append("\n");
return;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/monitor/jvm/JvmStats.java b/fe/fe-core/src/main/java/org/apache/doris/monitor/jvm/JvmStats.java
index a758f36..a298759 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/monitor/jvm/JvmStats.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/monitor/jvm/JvmStats.java
@@ -29,6 +29,7 @@ import java.lang.management.MemoryPoolMXBean;
import java.lang.management.MemoryUsage;
import java.lang.management.RuntimeMXBean;
import java.lang.management.ThreadMXBean;
+import java.lang.management.ThreadInfo;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -82,7 +83,28 @@ public class JvmStats {
}
Mem mem = new Mem(heapCommitted, heapUsed, heapMax, nonHeapCommitted, nonHeapUsed,
Collections.unmodifiableList(pools));
- Threads threads = new Threads(threadMXBean.getThreadCount(), threadMXBean.getPeakThreadCount());
+
+ int threadsNew = 0;
+ int threadsRunnable = 0;
+ int threadsBlocked = 0;
+ int threadsWaiting = 0;
+ int threadsTimedWaiting = 0;
+ int threadsTerminated = 0;
+ long threadIds[] = threadMXBean.getAllThreadIds();
+ for (ThreadInfo threadInfo : threadMXBean.getThreadInfo(threadIds, 0)) {
+ if (threadInfo == null) continue; // race protection
+ switch (threadInfo.getThreadState()) {
+ case NEW: threadsNew++; break;
+ case RUNNABLE: threadsRunnable++; break;
+ case BLOCKED: threadsBlocked++; break;
+ case WAITING: threadsWaiting++; break;
+ case TIMED_WAITING: threadsTimedWaiting++; break;
+ case TERMINATED: threadsTerminated++; break;
+ default: break;
+ }
+ }
+ Threads threads = new Threads(threadMXBean.getThreadCount(), threadMXBean.getPeakThreadCount(), threadsNew,
+ threadsRunnable, threadsBlocked, threadsWaiting, threadsTimedWaiting, threadsTerminated);
List<GarbageCollectorMXBean> gcMxBeans = ManagementFactory.getGarbageCollectorMXBeans();
GarbageCollector[] collectors = new GarbageCollector[gcMxBeans.size()];
@@ -192,6 +214,12 @@ public class JvmStats {
static final String THREADS = "threads";
static final String COUNT = "count";
static final String PEAK_COUNT = "peak_count";
+ static final String NEW_COUNT = "new_count";
+ static final String RUNNING_COUNT = "runnable_count";
+ static final String BLOCKED_COUNT = "blocked_count";
+ static final String WAITING_COUNT = "waiting_count";
+ static final String TIMED_WAITING_COUNT = "timed_waiting_count";
+ static final String TERMINATED_COUNT = "terminated_count";
static final String GC = "gc";
static final String COLLECTORS = "collectors";
@@ -264,10 +292,23 @@ public class JvmStats {
private final int count;
private final int peakCount;
-
- public Threads(int count, int peakCount) {
+ private final int threadsNewCount;
+ private final int threadsRunnableCount;
+ private final int threadsBlockedCount;
+ private final int threadsWaitingCount;
+ private final int threadsTimedWaitingCount;
+ private final int threadsTerminatedCount;
+
+ public Threads(int count, int peakCount, int threadsNewCount, int threadsRunnableCount, int threadsBlockedCount,
+ int threadsWaitingCount, int threadsTimedWaitingCount, int threadsTerminatedCount) {
this.count = count;
this.peakCount = peakCount;
+ this.threadsNewCount = threadsNewCount;
+ this.threadsRunnableCount = threadsRunnableCount;
+ this.threadsBlockedCount = threadsBlockedCount;
+ this.threadsWaitingCount = threadsWaitingCount;
+ this.threadsTimedWaitingCount = threadsTimedWaitingCount;
+ this.threadsTerminatedCount = threadsTerminatedCount;
}
public int getCount() {
@@ -278,6 +319,30 @@ public class JvmStats {
return peakCount;
}
+ public int getThreadsNewCount() {
+ return threadsNewCount;
+ }
+
+ public int getThreadsRunnableCount() {
+ return threadsRunnableCount;
+ }
+
+ public int getThreadsBlockedCount() {
+ return threadsBlockedCount;
+ }
+
+ public int getThreadsWaitingCount() {
+ return threadsWaitingCount;
+ }
+
+ public int getThreadsTimedWaitingCount() {
+ return threadsTimedWaitingCount;
+ }
+
+ public int getThreadsTerminatedCount() {
+ return threadsTerminatedCount;
+ }
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org