[GitHub] [incubator-doris] ccoffline commented on a change in pull request #6567: [Performance] Improve show proc statistic #6477

[GitBox <gi...@apache.org>]

ccoffline commented on a change in pull request #6567:
URL: https://github.com/apache/incubator-doris/pull/6567#discussion_r708827801



##########
File path: fe/fe-core/src/main/java/org/apache/doris/common/proc/StatisticProcDir.java
##########
@@ -58,86 +60,95 @@
 
     private Catalog catalog;
 
-    // db id -> set(tablet id)
-    Multimap<Long, Long> unhealthyTabletIds;
-    // db id -> set(tablet id)
-    Multimap<Long, Long> inconsistentTabletIds;
-    // db id -> set(tablet id)
-    Multimap<Long, Long> cloningTabletIds;
-    // db id -> set(tablet id)
-    Multimap<Long, Long> unrecoverableTabletIds;
-
     public StatisticProcDir(Catalog catalog) {
+        Preconditions.checkNotNull(catalog);
         this.catalog = catalog;
-        unhealthyTabletIds = HashMultimap.create();
-        inconsistentTabletIds = HashMultimap.create();
-        cloningTabletIds = HashMultimap.create();
-        unrecoverableTabletIds = HashMultimap.create();
     }
 
     @Override
     public ProcResult fetchResult() throws AnalysisException {
-        Preconditions.checkNotNull(catalog);
+        List<DBStatistic> statistics = catalog.getDbIds().parallelStream()
+                // skip information_schema database
+                .flatMap(id -> Stream.of(id == 0 ? null : catalog.getDbNullable(id)))
+                .filter(Objects::nonNull).map(DBStatistic::new)
+                // sort by dbName
+                .sorted(Comparator.comparing(db -> db.db.getFullName()))
+                .collect(Collectors.toList());
+
+        List<List<String>> rows = new ArrayList<>(statistics.size() + 1);
+        for (DBStatistic statistic : statistics) {
+            rows.add(statistic.toRow());
+        }
+        rows.add(statistics.stream().reduce(new DBStatistic(), DBStatistic::reduce).toRow());
 
-        BaseProcResult result = new BaseProcResult();
+        return new BaseProcResult(TITLE_NAMES, rows);
+    }
+
+    @Override
+    public boolean register(String name, ProcNodeInterface node) {
+        return false;
+    }
 
-        result.setNames(TITLE_NAMES);
-        List<Long> dbIds = catalog.getDbIds();
-        if (dbIds == null || dbIds.isEmpty()) {
-            // empty
-            return result;
+    @Override
+    public ProcNodeInterface lookup(String dbIdStr) throws AnalysisException {
+        try {
+            long dbId = Long.parseLong(dbIdStr);
+            return catalog.getDb(dbId).map(IncompleteTabletsProcNode::new).orElse(null);
+        } catch (NumberFormatException e) {
+            throw new AnalysisException("Invalid db id format: " + dbIdStr);
         }
+    }
 
-        SystemInfoService infoService = Catalog.getCurrentSystemInfo();
-
-        int totalDbNum = 0;
-        int totalTableNum = 0;
-        int totalPartitionNum = 0;
-        int totalIndexNum = 0;
-        int totalTabletNum = 0;
-        int totalReplicaNum = 0;
-
-        unhealthyTabletIds.clear();
-        inconsistentTabletIds.clear();
-        cloningTabletIds = AgentTaskQueue.getTabletIdsByType(TTaskType.CLONE);
-        List<List<Comparable>> lines = new ArrayList<List<Comparable>>();
-        for (Long dbId : dbIds) {
-            if (dbId == 0) {
-                // skip information_schema database
-                continue;
-            }
-            Database db = catalog.getDbNullable(dbId);
-            if (db == null) {
-                continue;
-            }
+    static class DBStatistic {
+        boolean summary;
+        Database db;
+        int dbNum;
+        int tableNum;
+        int partitionNum;
+        int indexNum;
+        int tabletNum;
+        int replicaNum;
+        int unhealthyTabletNum;
+        int inconsistentTabletNum;
+        int cloningTabletNum;
+        int badTabletNum;
+        Set<Long> unhealthyTabletIds;
+        Set<Long> inconsistentTabletIds;
+        Set<Long> cloningTabletIds;
+        Set<Long> unrecoverableTabletIds;
+
+        DBStatistic() {
+            this.summary = true;
+        }
 
-            ++totalDbNum;
+        DBStatistic(Database db) {
+            Preconditions.checkNotNull(db);
+            this.summary = false;
+            this.db = db;
+            this.dbNum = 1;
+            this.unhealthyTabletIds = new HashSet<>();
+            this.inconsistentTabletIds = new HashSet<>();
+            this.unrecoverableTabletIds = new HashSet<>();
+            this.cloningTabletIds = AgentTaskQueue.getTask(db.getId(), TTaskType.CLONE)
+                    .stream().map(AgentTask::getTabletId).collect(Collectors.toSet());
+
+            SystemInfoService infoService = Catalog.getCurrentSystemInfo();
             List<Long> aliveBeIdsInCluster = infoService.getClusterBackendIds(db.getClusterName(), true);
             db.readLock();

Review comment:
       ok




-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org



---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org