You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zh...@apache.org on 2020/06/09 03:24:51 UTC
[hbase] branch branch-2.2 updated: HBASE-24117 Shutdown
AssignmentManager before ProcedureExecutor may cause SCP to accidentally
skip assigning a region (#1865)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-2.2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.2 by this push:
new dee9933 HBASE-24117 Shutdown AssignmentManager before ProcedureExecutor may cause SCP to accidentally skip assigning a region (#1865)
dee9933 is described below
commit dee99336dda831e1fa69e7db0cb5bf01c8f480f7
Author: Duo Zhang <zh...@apache.org>
AuthorDate: Tue Jun 9 11:07:16 2020 +0800
HBASE-24117 Shutdown AssignmentManager before ProcedureExecutor may cause SCP to accidentally skip assigning a region (#1865)
Signed-off-by: Michael Stack <st...@apache.org>
---
.../src/main/java/org/apache/hadoop/hbase/master/HMaster.java | 7 +++++--
.../apache/hadoop/hbase/master/procedure/ServerCrashProcedure.java | 7 +++++++
2 files changed, 12 insertions(+), 2 deletions(-)
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index e9b22a6..b9b7076 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -1464,6 +1464,11 @@ public class HMaster extends HRegionServer implements MasterServices {
LOG.debug("Stopping service threads");
+ // stop procedure executor prior to other services such as server manager and assignment
+ // manager, as these services are important for some running procedures. See HBASE-24117 for
+ // example.
+ stopProcedureExecutor();
+
if (this.quotaManager != null) {
this.quotaManager.stop();
}
@@ -1478,8 +1483,6 @@ public class HMaster extends HRegionServer implements MasterServices {
this.assignmentManager.stop();
}
- stopProcedureExecutor();
-
if (this.walManager != null) {
this.walManager.stop();
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.java
index 916acfe..a2e7563 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.java
@@ -24,6 +24,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import org.apache.hadoop.hbase.DoNotRetryIOException;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.RegionInfoBuilder;
@@ -473,6 +474,12 @@ public class ServerCrashProcedure
// updated the region location, or even finished itself, so the region is no longer on us
// any more, we should not try to assign it again. Please see HBASE-23594 for more details.
if (!serverName.equals(regionNode.getRegionLocation())) {
+ // See HBASE-24117, though we have already changed the shutdown order, it is still worth
+ // double checking here to confirm that we do not skip assignment incorrectly.
+ if (!am.isRunning()) {
+ throw new DoNotRetryIOException(
+ "AssignmentManager has been stopped, can not process assignment any more");
+ }
LOG.info("{} found a region {} which is no longer on us {}, give up assigning...", this,
regionNode, serverName);
continue;