[Metrics] Add tablet count on different status metrics

Based on these metrics, we can add alerts to remind admins whether the cluster is healthy or not.
acelyc111 · May 11, 2021 · e973382 · e973382
1 parent c6bbc68
commit e973382
Show file tree

Hide file tree

Showing 2 changed files with 37 additions and 3 deletions.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletChecker.java b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletChecker.java
@@ -35,6 +35,10 @@
 import org.apache.doris.common.DdlException;
 import org.apache.doris.common.Pair;
 import org.apache.doris.common.util.MasterDaemon;
+import org.apache.doris.metric.GaugeMetric;
+import org.apache.doris.metric.Metric;
+import org.apache.doris.metric.MetricLabel;
+import org.apache.doris.metric.MetricRepo;
 import org.apache.doris.system.SystemInfoService;
 
 import com.google.common.base.Preconditions;
@@ -46,10 +50,12 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
+import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.stream.Collectors;
 
 /*
@@ -66,6 +72,14 @@ public class TabletChecker extends MasterDaemon {
     private TabletScheduler tabletScheduler;
     private TabletSchedulerStat stat;
 
+    HashMap<String, AtomicLong> tabletCountByStatus = new HashMap<String, AtomicLong>(){{
+        put("total", new AtomicLong(0L));
+        put("unhealthy", new AtomicLong(0L));
+        put("added", new AtomicLong(0L));
+        put("in_sched", new AtomicLong(0L));
+        put("not_ready", new AtomicLong(0L));
+    }};
+
     // db id -> (tbl id -> PrioPart)
     // priority of replicas of partitions in this table will be set to VERY_HIGH if not healthy
     private com.google.common.collect.Table<Long, Long, Set<PrioPart>> prios = HashBasedTable.create();
@@ -119,6 +133,22 @@ public TabletChecker(Catalog catalog, SystemInfoService infoService, TabletSched
         this.infoService = infoService;
         this.tabletScheduler = tabletScheduler;
         this.stat = stat;
+
+        initMetrics();
+    }
+
+    private void initMetrics() {
+        for (String status : tabletCountByStatus.keySet()) {
+            GaugeMetric<Long> gauge = new GaugeMetric<Long>("tablet_status_count",
+                    Metric.MetricUnit.NOUNIT, "tablet count on different status") {
+                @Override
+                public Long getValue() {
+                    return tabletCountByStatus.get(status).get();
+                }
+            };
+            gauge.addLabel(new MetricLabel("type", status));
+            MetricRepo.PALO_METRIC_REGISTER.addPaloMetrics(gauge);
+        }
     }
 
     private void addPrios(RepairTabletInfo repairTabletInfo, long timeoutMs) {
@@ -229,12 +259,10 @@ private void checkTablets() {
                         for (MaterializedIndex idx : partition.getMaterializedIndices(IndexExtState.VISIBLE)) {
                             for (Tablet tablet : idx.getTablets()) {
                                 totalTabletNum++;
-
                                 if (tabletScheduler.containsTablet(tablet.getId())) {
                                     tabletInScheduler++;
                                     continue;
                                 }
-
                                 Pair<TabletStatus, TabletSchedCtx.Priority> statusWithPrio = tablet.getHealthStatusWithPriority(
                                         infoService,
                                         db.getClusterName(),
@@ -302,6 +330,12 @@ private void checkTablets() {
         stat.counterUnhealthyTabletNum.addAndGet(unhealthyTabletNum);
         stat.counterTabletAddToBeScheduled.addAndGet(addToSchedulerTabletNum);
 
+        tabletCountByStatus.get("unhealthy").set(unhealthyTabletNum);
+        tabletCountByStatus.get("total").set(totalTabletNum);
+        tabletCountByStatus.get("added").set(addToSchedulerTabletNum);
+        tabletCountByStatus.get("in_sched").set(tabletInScheduler);
+        tabletCountByStatus.get("not_ready").set(tabletNotReady);
+
         LOG.info("finished to check tablets. unhealth/total/added/in_sched/not_ready: {}/{}/{}/{}/{}, cost: {} ms",
                 unhealthyTabletNum, totalTabletNum, addToSchedulerTabletNum, tabletInScheduler, tabletNotReady, cost);
     }

diff --git a/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java b/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java
@@ -54,7 +54,7 @@ public final class MetricRepo {
     private static final Logger LOG = LogManager.getLogger(MetricRepo.class);
 
     private static final MetricRegistry METRIC_REGISTER = new MetricRegistry();
-    private static final DorisMetricRegistry PALO_METRIC_REGISTER = new DorisMetricRegistry();
+    public static final DorisMetricRegistry PALO_METRIC_REGISTER = new DorisMetricRegistry();
 
     public static volatile boolean isInit = false;
     public static final SystemMetrics SYSTEM_METRICS = new SystemMetrics();