[GitHub] [hbase] Apache9 commented on a change in pull request #3862: HBASE-26468 Region Server doesn't exit cleanly incase it crashes.

[GitBox <gi...@apache.org>]

Apache9 commented on a change in pull request #3862:
URL: https://github.com/apache/hbase/pull/3862#discussion_r757596948



##########
File path: hbase-server/src/main/java/org/apache/hadoop/hbase/util/ServerCommandLine.java
##########
@@ -141,18 +142,51 @@ public static void logProcessInfo(Configuration conf) {
   }
 
   /**
-   * Parse and run the given command line. This may exit the JVM if
-   * a nonzero exit code is returned from <code>run()</code>.
+   * Parse and run the given command line. This will exit the JVM with
+   * the exit code returned from <code>run()</code>.
+   * If return code is 0, wait for atmost 30 seconds for all non-daemon threads to quit,
+   * otherwise exit the jvm
    */
   public void doMain(String args[]) {
     try {
       int ret = ToolRunner.run(HBaseConfiguration.create(), this, args);
       if (ret != 0) {
         System.exit(ret);
       }
+      // Return code is 0 here.
+      boolean forceStop = false;
+      long now = EnvironmentEdgeManager.currentTime();
+      while (isNonDaemonThreadRunning()) {
+        if (EnvironmentEdgeManager.currentTime() - now > 30 * 1000) {
+          forceStop = true;
+          break;
+        }
+        Thread.sleep(1000);
+      }
+      if (forceStop) {
+        LOG.error("Failed to stop all non-daemon threads, so force quitting");
+        System.exit(-1);
+      }
     } catch (Exception e) {
       LOG.error("Failed to run", e);
       System.exit(-1);
     }
   }
+
+  /**
+   * Checks whether any non-daemon thread is running.
+   * @return true if there are non daemon threads running, otherwise false
+   */
+  public boolean isNonDaemonThreadRunning() {

Review comment:
       Move this method to Threads class under hbase-common?

##########
File path: hbase-server/src/main/java/org/apache/hadoop/hbase/util/ServerCommandLine.java
##########
@@ -141,18 +142,51 @@ public static void logProcessInfo(Configuration conf) {
   }
 
   /**
-   * Parse and run the given command line. This may exit the JVM if
-   * a nonzero exit code is returned from <code>run()</code>.
+   * Parse and run the given command line. This will exit the JVM with
+   * the exit code returned from <code>run()</code>.
+   * If return code is 0, wait for atmost 30 seconds for all non-daemon threads to quit,
+   * otherwise exit the jvm
    */
   public void doMain(String args[]) {
     try {
       int ret = ToolRunner.run(HBaseConfiguration.create(), this, args);
       if (ret != 0) {
         System.exit(ret);
       }
+      // Return code is 0 here.
+      boolean forceStop = false;
+      long now = EnvironmentEdgeManager.currentTime();
+      while (isNonDaemonThreadRunning()) {
+        if (EnvironmentEdgeManager.currentTime() - now > 30 * 1000) {
+          forceStop = true;
+          break;
+        }
+        Thread.sleep(1000);
+      }
+      if (forceStop) {
+        LOG.error("Failed to stop all non-daemon threads, so force quitting");
+        System.exit(-1);
+      }
     } catch (Exception e) {
       LOG.error("Failed to run", e);
       System.exit(-1);
     }
   }
+
+  /**
+   * Checks whether any non-daemon thread is running.
+   * @return true if there are non daemon threads running, otherwise false
+   */
+  public boolean isNonDaemonThreadRunning() {
+    AtomicInteger nonDaemonThreadCount = new AtomicInteger();
+    Set<Thread> threads =  Thread.getAllStackTraces().keySet();
+    threads.forEach(t -> {
+      // Exclude current thread
+      if (t.getId() != Thread.currentThread().getId() && !t.isDaemon()) {
+        nonDaemonThreadCount.getAndIncrement();
+        LOG.info("Non daemon thread name: " + t.getName() + "  still alive");

Review comment:
       It will be good if we could print the stack trace here.




-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@hbase.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org