This is an automated email from the ASF dual-hosted git repository.
binlijin pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new 5a0dd57 HBASE-23615 Use a dedicated thread for executing WorkerMonitor in Pro… (#961)
5a0dd57 is described below
commit 5a0dd574a6f86c158ea8d3ff9a157462d110c037
Author: binlijin <bi...@gmail.com>
AuthorDate: Tue Dec 31 10:03:01 2019 +0800
HBASE-23615 Use a dedicated thread for executing WorkerMonitor in Pro… (#961)
Signed-off-by: stack <st...@apache.org>
Signed-off-by: Duo Zhang <zh...@apache.org>
Signed-off-by: virajjasani <34...@users.noreply.github.com>
---
.../hadoop/hbase/procedure2/ProcedureExecutor.java | 18 ++++++++++++++++--
.../hadoop/hbase/procedure2/TimeoutExecutorThread.java | 5 +++--
.../hbase/master/procedure/TestProcedureAdmin.java | 1 -
3 files changed, 19 insertions(+), 5 deletions(-)
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
index 93fca64..c680ade 100644
--- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
@@ -217,6 +217,15 @@ public class ProcedureExecutor<TEnvironment> {
*/
private TimeoutExecutorThread<TEnvironment> timeoutExecutor;
+ /**
+ * WorkerMonitor check for stuck workers and new worker thread when necessary, for example if
+ * there is no worker to assign meta, it will new worker thread for it, so it is very important.
+ * TimeoutExecutor execute many tasks like DeadServerMetricRegionChore RegionInTransitionChore
+ * and so on, some tasks may execute for a long time so will block other tasks like
+ * WorkerMonitor, so use a dedicated thread for executing WorkerMonitor.
+ */
+ private TimeoutExecutorThread<TEnvironment> workerMonitorExecutor;
+
private int corePoolSize;
private int maxPoolSize;
@@ -560,7 +569,8 @@ public class ProcedureExecutor<TEnvironment> {
corePoolSize, maxPoolSize);
this.threadGroup = new ThreadGroup("PEWorkerGroup");
- this.timeoutExecutor = new TimeoutExecutorThread<>(this, threadGroup);
+ this.timeoutExecutor = new TimeoutExecutorThread<>(this, threadGroup, "ProcExecTimeout");
+ this.workerMonitorExecutor = new TimeoutExecutorThread<>(this, threadGroup, "WorkerMonitor");
// Create the workers
workerId.set(0);
@@ -604,12 +614,13 @@ public class ProcedureExecutor<TEnvironment> {
// Start the executors. Here we must have the lastProcId set.
LOG.trace("Start workers {}", workerThreads.size());
timeoutExecutor.start();
+ workerMonitorExecutor.start();
for (WorkerThread worker: workerThreads) {
worker.start();
}
// Internal chores
- timeoutExecutor.add(new WorkerMonitor());
+ workerMonitorExecutor.add(new WorkerMonitor());
// Add completed cleaner chore
addChore(new CompletedProcedureCleaner<>(conf, store, procExecutionLock, completed,
@@ -624,6 +635,7 @@ public class ProcedureExecutor<TEnvironment> {
LOG.info("Stopping");
scheduler.stop();
timeoutExecutor.sendStopSignal();
+ workerMonitorExecutor.sendStopSignal();
}
@VisibleForTesting
@@ -632,6 +644,8 @@ public class ProcedureExecutor<TEnvironment> {
// stop the timeout executor
timeoutExecutor.awaitTermination();
+ // stop the work monitor executor
+ workerMonitorExecutor.awaitTermination();
// stop the worker threads
for (WorkerThread worker: workerThreads) {
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/TimeoutExecutorThread.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/TimeoutExecutorThread.java
index 30a52d4..1e796d9 100644
--- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/TimeoutExecutorThread.java
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/TimeoutExecutorThread.java
@@ -37,8 +37,9 @@ class TimeoutExecutorThread<TEnvironment> extends StoppableThread {
private final DelayQueue<DelayedWithTimeout> queue = new DelayQueue<>();
- public TimeoutExecutorThread(ProcedureExecutor<TEnvironment> executor, ThreadGroup group) {
- super(group, "ProcExecTimeout");
+ public TimeoutExecutorThread(ProcedureExecutor<TEnvironment> executor, ThreadGroup group,
+ String name) {
+ super(group, name);
setDaemon(true);
this.executor = executor;
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestProcedureAdmin.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestProcedureAdmin.java
index 02f0257..b040c71 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestProcedureAdmin.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestProcedureAdmin.java
@@ -57,7 +57,6 @@ public class TestProcedureAdmin {
protected static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
-
private static void setupConf(Configuration conf) {
conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1);
}