You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2021/06/30 01:25:59 UTC
[incubator-doris] branch master updated: [Metrics] Add tablet count
on different status metrics (#5787)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new e3899aa [Metrics] Add tablet count on different status metrics (#5787)
e3899aa is described below
commit e3899aa3e7dab479883030c5a56223f940f536a3
Author: Yingchun Lai <40...@qq.com>
AuthorDate: Wed Jun 30 09:25:42 2021 +0800
[Metrics] Add tablet count on different status metrics (#5787)
Based on these metrics, we can add alerts to remind
admins whether the cluster is healthy or not.
---
.../java/org/apache/doris/clone/TabletChecker.java | 37 ++++++++++++++++++++++
.../java/org/apache/doris/metric/MetricRepo.java | 2 +-
2 files changed, 38 insertions(+), 1 deletion(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletChecker.java b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletChecker.java
index 2c9666d..77238c4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletChecker.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletChecker.java
@@ -35,6 +35,10 @@ import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.Pair;
import org.apache.doris.common.util.MasterDaemon;
+import org.apache.doris.metric.GaugeMetric;
+import org.apache.doris.metric.Metric;
+import org.apache.doris.metric.MetricLabel;
+import org.apache.doris.metric.MetricRepo;
import org.apache.doris.system.SystemInfoService;
import com.google.common.base.Preconditions;
@@ -46,10 +50,12 @@ import com.google.common.collect.Table.Cell;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Collectors;
/*
@@ -66,6 +72,14 @@ public class TabletChecker extends MasterDaemon {
private TabletScheduler tabletScheduler;
private TabletSchedulerStat stat;
+ HashMap<String, AtomicLong> tabletCountByStatus = new HashMap<String, AtomicLong>(){{
+ put("total", new AtomicLong(0L));
+ put("unhealthy", new AtomicLong(0L));
+ put("added", new AtomicLong(0L));
+ put("in_sched", new AtomicLong(0L));
+ put("not_ready", new AtomicLong(0L));
+ }};
+
// db id -> (tbl id -> PrioPart)
// priority of replicas of partitions in this table will be set to VERY_HIGH if not healthy
private com.google.common.collect.Table<Long, Long, Set<PrioPart>> prios = HashBasedTable.create();
@@ -119,6 +133,22 @@ public class TabletChecker extends MasterDaemon {
this.infoService = infoService;
this.tabletScheduler = tabletScheduler;
this.stat = stat;
+
+ initMetrics();
+ }
+
+ private void initMetrics() {
+ for (String status : tabletCountByStatus.keySet()) {
+ GaugeMetric<Long> gauge = new GaugeMetric<Long>("tablet_status_count",
+ Metric.MetricUnit.NOUNIT, "tablet count on different status") {
+ @Override
+ public Long getValue() {
+ return tabletCountByStatus.get(status).get();
+ }
+ };
+ gauge.addLabel(new MetricLabel("type", status));
+ MetricRepo.PALO_METRIC_REGISTER.addPaloMetrics(gauge);
+ }
}
private void addPrios(RepairTabletInfo repairTabletInfo, long timeoutMs) {
@@ -272,6 +302,7 @@ public class TabletChecker extends MasterDaemon {
if (isInPrios(db.getId(), tbl.getId(), partition.getId())) {
continue;
}
+
LoopControlStatus st = handlePartitionTablet(db, tbl, partition, false,
aliveBeIdsInCluster, start, counter);
if (st == LoopControlStatus.BREAK_OUT) {
@@ -292,6 +323,12 @@ public class TabletChecker extends MasterDaemon {
stat.counterUnhealthyTabletNum.addAndGet(counter.unhealthyTabletNum);
stat.counterTabletAddToBeScheduled.addAndGet(counter.addToSchedulerTabletNum);
+ tabletCountByStatus.get("unhealthy").set(counter.unhealthyTabletNum);
+ tabletCountByStatus.get("total").set(counter.totalTabletNum);
+ tabletCountByStatus.get("added").set(counter.addToSchedulerTabletNum);
+ tabletCountByStatus.get("in_sched").set(counter.tabletInScheduler);
+ tabletCountByStatus.get("not_ready").set(counter.tabletNotReady);
+
LOG.info("finished to check tablets. unhealth/total/added/in_sched/not_ready: {}/{}/{}/{}/{}, cost: {} ms",
counter.unhealthyTabletNum, counter.totalTabletNum, counter.addToSchedulerTabletNum,
counter.tabletInScheduler, counter.tabletNotReady, cost);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java b/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java
index 2e26c70..9196eea 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java
@@ -54,7 +54,7 @@ public final class MetricRepo {
private static final Logger LOG = LogManager.getLogger(MetricRepo.class);
private static final MetricRegistry METRIC_REGISTER = new MetricRegistry();
- private static final DorisMetricRegistry PALO_METRIC_REGISTER = new DorisMetricRegistry();
+ public static final DorisMetricRegistry PALO_METRIC_REGISTER = new DorisMetricRegistry();
public static volatile boolean isInit = false;
public static final SystemMetrics SYSTEM_METRICS = new SystemMetrics();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org