You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by pa...@apache.org on 2021/01/04 15:34:55 UTC
[hbase] branch master updated: HBASE-25447 remoteProc is suspended
due to OOM ERROR (#2824)
This is an automated email from the ASF dual-hosted git repository.
pankajkumar pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new 600be60 HBASE-25447 remoteProc is suspended due to OOM ERROR (#2824)
600be60 is described below
commit 600be60a4bd4d3b3e9652027a0cb8bdd32016c6b
Author: Bo Cui <cu...@163.com>
AuthorDate: Mon Jan 4 23:34:38 2021 +0800
HBASE-25447 remoteProc is suspended due to OOM ERROR (#2824)
Some OMME can not cause the JVM to exit, like "java.lang.OutOfMemoryError: Direct buffer memory", "java.lang.OutOfMemoryError: unable to create new native thread", as they dont call vmError#next_OnError_command. So abort HMaster when uncaught exception occurs in TimeoutExecutor, the new active Hmaster will resume the suspended procedure.
Signed-off-by: Duo Zhang <zh...@apache.org>
Signed-off-by: stack <st...@apache.com>
Signed-off-by: Pankaj Kumar<pa...@apache.org>
---
.../apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java | 4 ++++
.../hadoop/hbase/master/procedure/RSProcedureDispatcher.java | 8 ++++++++
2 files changed, 12 insertions(+)
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java
index 611fc86..a060f14 100644
--- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java
@@ -106,6 +106,10 @@ public abstract class RemoteProcedureDispatcher<TEnv, TRemote extends Comparable
return true;
}
+ protected void setTimeoutExecutorUncaughtExceptionHandler(UncaughtExceptionHandler eh) {
+ timeoutExecutor.setUncaughtExceptionHandler(eh);
+ }
+
public boolean stop() {
if (!running.getAndSet(false)) {
return false;
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
index 2f990cb..d028bb4 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
@@ -94,6 +94,7 @@ public class RSProcedureDispatcher
if (!super.start()) {
return false;
}
+ setTimeoutExecutorUncaughtExceptionHandler(this::abort);
if (master.isStopped()) {
LOG.debug("Stopped");
return false;
@@ -126,6 +127,13 @@ public class RSProcedureDispatcher
return true;
}
+ private void abort(Thread t, Throwable e) {
+ LOG.error("Caught error", e);
+ if (!master.isStopped() && !master.isStopping() && !master.isAborted()) {
+ master.abort("Aborting master", e);
+ }
+ }
+
@Override
public boolean stop() {
if (!super.stop()) {