[iotdb] branch apache_0.12_0630_apply_stuck updated: debug apply stuck

[ne...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

neuyilan pushed a commit to branch apache_0.12_0630_apply_stuck
in repository https://gitbox.apache.org/repos/asf/iotdb.git


The following commit(s) were added to refs/heads/apache_0.12_0630_apply_stuck by this push:
     new 23c6150  debug apply stuck
23c6150 is described below

commit 23c6150c9b171850494022c8da53d625267f7c04
Author: HouliangQi <ne...@163.com>
AuthorDate: Wed Jun 30 18:04:57 2021 +0800

    debug apply stuck
---
 .../iotdb/cluster/log/manage/RaftLogManager.java   | 39 ++++++++++++++++++++++
 1 file changed, 39 insertions(+)

diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/log/manage/RaftLogManager.java b/cluster/src/main/java/org/apache/iotdb/cluster/log/manage/RaftLogManager.java
index 446eefc..a46bca2 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/log/manage/RaftLogManager.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/log/manage/RaftLogManager.java
@@ -38,7 +38,14 @@ import org.apache.commons.lang3.concurrent.BasicThreadFactory;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
 import java.io.IOException;
+import java.io.InputStreamReader;
+import java.lang.management.ManagementFactory;
+import java.lang.management.RuntimeMXBean;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
@@ -943,10 +950,36 @@ public abstract class RaftLogManager {
             nextToCheckIndex);
         return;
       }
+      long startTime = System.currentTimeMillis();
+      long lastDump = Long.MAX_VALUE;
       synchronized (log) {
         while (!log.isApplied() && maxHaveAppliedCommitIndex < log.getCurrLogIndex()) {
           // wait until the log is applied or a newer snapshot is installed
           log.wait(5);
+
+          long now = System.currentTimeMillis();
+          if ((now - startTime) > 10000 && now - lastDump > 2000) {
+            int processId = getProcessID();
+            String command = "jstack -l " + processId;
+            try {
+              Process proc = Runtime.getRuntime().exec(command);
+              BufferedReader reader =
+                  new BufferedReader(new InputStreamReader(proc.getInputStream()));
+              String fileName = System.getProperty("user.dir") + File.separator + "jstack_" + now;
+              BufferedWriter out = new BufferedWriter(new FileWriter(fileName));
+              String str;
+              while ((str = reader.readLine()) != null) {
+                out.write(str);
+                out.newLine();
+              }
+              out.close();
+              reader.close();
+              proc.waitFor();
+            } catch (IOException e) {
+              logger.error("dump thread failed, e", e);
+            }
+            lastDump = System.currentTimeMillis();
+          }
         }
       }
       synchronized (changeApplyCommitIndexCond) {
@@ -978,6 +1011,12 @@ public abstract class RaftLogManager {
     }
   }
 
+  public int getProcessID() {
+    RuntimeMXBean runtimeMXBean = ManagementFactory.getRuntimeMXBean();
+    System.out.println(runtimeMXBean.getName());
+    return Integer.valueOf(runtimeMXBean.getName().split("@")[0]).intValue();
+  }
+
   /**
    * Clear the fence that blocks the application of new logs, and continue to apply the cached
    * unapplied logs.