You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zh...@apache.org on 2020/11/13 08:23:42 UTC
[hbase] branch branch-2.3 updated: HBASE-25255 Master fails to
initialize when creating rs group table (#2638)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-2.3
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.3 by this push:
new b5b1c37 HBASE-25255 Master fails to initialize when creating rs group table (#2638)
b5b1c37 is described below
commit b5b1c3743337674b89123072f9939bab2b01fdb2
Author: Duo Zhang <zh...@apache.org>
AuthorDate: Fri Nov 13 14:28:31 2020 +0800
HBASE-25255 Master fails to initialize when creating rs group table (#2638)
Signed-off-by: Guanghao Zhang <zg...@apache.org>
---
.../org/apache/hadoop/hbase/master/HMaster.java | 8 ++---
.../hbase/master/assignment/AssignmentManager.java | 35 ++++++++++++++++------
.../master/procedure/CreateTableProcedure.java | 5 ++--
.../procedure/MasterProcedureTestingUtility.java | 1 +
4 files changed, 33 insertions(+), 16 deletions(-)
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 1a394f9..f758b8b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -1073,12 +1073,10 @@ public class HMaster extends HRegionServer implements MasterServices {
this.assignmentManager.joinCluster();
// The below depends on hbase:meta being online.
this.tableStateManager.start();
- // Below has to happen after tablestatemanager has started in the case where this hbase-2.x
- // is being started over an hbase-1.x dataset. tablestatemanager runs a migration as part
- // of its 'start' moving table state from zookeeper to hbase:meta. This migration needs to
- // complete before we do this next step processing offline regions else it fails reading
- // table states messing up master launch (namespace table, etc., are not assigned).
this.assignmentManager.processOfflineRegions();
+ // this must be called after the above processOfflineRegions to prevent race
+ this.assignmentManager.wakeMetaLoadedEvent();
+
// Initialize after meta is up as below scans meta
if (favoredNodesManager != null && !maintenanceMode) {
SnapshotOfRegionAssignmentFromMeta snapshotOfRegionAssignment =
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java
index 68c9ad6..a4a34c7 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java
@@ -487,8 +487,16 @@ public class AssignmentManager {
return metaLoadEvent.suspendIfNotReady(proc);
}
+ /**
+ * This method will be called in master initialization method after calling
+ * {@link #processOfflineRegions()}, as in processOfflineRegions we will generate assign
+ * procedures for offline regions, which may be conflict with creating table.
+ * <p/>
+ * This is a bit dirty, should be reconsidered after we decide whether to keep the
+ * {@link #processOfflineRegions()} method.
+ */
@VisibleForTesting
- void wakeMetaLoadedEvent() {
+ public void wakeMetaLoadedEvent() {
metaLoadEvent.wake(getProcedureScheduler());
assert isMetaLoaded() : "expected meta to be loaded";
}
@@ -1466,12 +1474,23 @@ public class AssignmentManager {
// Public so can be run by the Master as part of the startup. Needs hbase:meta to be online.
// Needs to be done after the table state manager has been started.
public void processOfflineRegions() {
- List<RegionInfo> offlineRegions = regionStates.getRegionStates().stream()
- .filter(RegionState::isOffline).filter(s -> isTableEnabled(s.getRegion().getTable()))
- .map(RegionState::getRegion).collect(Collectors.toList());
- if (!offlineRegions.isEmpty()) {
- master.getMasterProcedureExecutor().submitProcedures(
- master.getAssignmentManager().createRoundRobinAssignProcedures(offlineRegions));
+ TransitRegionStateProcedure[] procs =
+ regionStates.getRegionStateNodes().stream().filter(rsn -> rsn.isInState(State.OFFLINE))
+ .filter(rsn -> isTableEnabled(rsn.getRegionInfo().getTable())).map(rsn -> {
+ rsn.lock();
+ try {
+ if (rsn.getProcedure() != null) {
+ return null;
+ } else {
+ return rsn.setProcedure(TransitRegionStateProcedure.assign(getProcedureEnvironment(),
+ rsn.getRegionInfo(), null));
+ }
+ } finally {
+ rsn.unlock();
+ }
+ }).filter(p -> p != null).toArray(TransitRegionStateProcedure[]::new);
+ if (procs.length > 0) {
+ master.getMasterProcedureExecutor().submitProcedures(procs);
}
}
@@ -1551,8 +1570,6 @@ public class AssignmentManager {
private void loadMeta() throws IOException {
// TODO: use a thread pool
regionStateStore.visitMeta(new RegionMetaLoadingVisitor());
- // every assignment is blocked until meta is loaded.
- wakeMetaLoadedEvent();
}
/**
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java
index 03ed6dc..8f3d0a8 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java
@@ -230,8 +230,9 @@ public class CreateTableProcedure
@Override
protected boolean waitInitialized(MasterProcedureEnv env) {
if (getTableName().isSystemTable()) {
- // Creating system table is part of the initialization, so do not wait here.
- return false;
+ // Creating system table is part of the initialization, so only wait for meta loaded instead
+ // of waiting for master fully initialized.
+ return env.getAssignmentManager().waitMetaLoaded(this);
}
return super.waitInitialized(env);
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
index c60a604..e6ea096 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
@@ -113,6 +113,7 @@ public class MasterProcedureTestingUtility {
AssignmentManager am = env.getAssignmentManager();
try {
am.joinCluster();
+ am.wakeMetaLoadedEvent();
master.setInitialized(true);
} catch (Exception e) {
LOG.warn("Failed to load meta", e);