You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by pa...@apache.org on 2021/01/06 18:21:01 UTC
[hbase] branch branch-2.3 updated: HBASE-25447 remoteProc is
suspended due to OOM ERROR (#2824)
This is an automated email from the ASF dual-hosted git repository.
pankajkumar pushed a commit to branch branch-2.3
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.3 by this push:
new d1ff84d HBASE-25447 remoteProc is suspended due to OOM ERROR (#2824)
d1ff84d is described below
commit d1ff84d076bebb1f6911c5579405e82ddee27628
Author: Bo Cui <cu...@163.com>
AuthorDate: Mon Jan 4 23:34:38 2021 +0800
HBASE-25447 remoteProc is suspended due to OOM ERROR (#2824)
Some OMME can not cause the JVM to exit, like "java.lang.OutOfMemoryError: Direct buffer memory", "java.lang.OutOfMemoryError: unable to create new native thread", as they dont call vmError#next_OnError_command. So abort HMaster when uncaught exception occurs in TimeoutExecutor, the new active Hmaster will resume the suspended procedure.
Signed-off-by: Duo Zhang <zh...@apache.org>
Signed-off-by: stack <st...@apache.com>
Signed-off-by: Pankaj Kumar<pa...@apache.org>
(cherry picked from commit 600be60a4bd4d3b3e9652027a0cb8bdd32016c6b)
(cherry picked from commit ae77f81e7eaf07e9943688f10f7f7e14211151c2)
---
.../apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java | 4 ++++
.../hadoop/hbase/master/procedure/RSProcedureDispatcher.java | 8 ++++++++
2 files changed, 12 insertions(+)
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java
index ecb4ffc..fadf0fb 100644
--- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java
@@ -105,6 +105,10 @@ public abstract class RemoteProcedureDispatcher<TEnv, TRemote extends Comparable
return true;
}
+ protected void setTimeoutExecutorUncaughtExceptionHandler(UncaughtExceptionHandler eh) {
+ timeoutExecutor.setUncaughtExceptionHandler(eh);
+ }
+
public boolean stop() {
if (!running.getAndSet(false)) {
return false;
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
index 3b4ee4c..c740fd8 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
@@ -94,6 +94,7 @@ public class RSProcedureDispatcher
if (!super.start()) {
return false;
}
+ setTimeoutExecutorUncaughtExceptionHandler(this::abort);
if (master.isStopped()) {
LOG.debug("Stopped");
return false;
@@ -126,6 +127,13 @@ public class RSProcedureDispatcher
return true;
}
+ private void abort(Thread t, Throwable e) {
+ LOG.error("Caught error", e);
+ if (!master.isStopped() && !master.isStopping() && !master.isAborted()) {
+ master.abort("Aborting master", e);
+ }
+ }
+
@Override
public boolean stop() {
if (!super.stop()) {