You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2017/08/08 21:02:20 UTC
hbase git commit: HBASE-18492 [AMv2] Embed code for selecting highest
versioned region server for system table regions in
AssignmentManager.processAssignQueue()
Repository: hbase
Updated Branches:
refs/heads/master 03390684c -> f314b5911
HBASE-18492 [AMv2] Embed code for selecting highest versioned region server for system table regions in AssignmentManager.processAssignQueue()
* Modified AssignmentManager.processAssignQueue() method to consider only highest versioned region servers for system table regions when
destination server is not specified for them. Destination server is retained, if specified.
* Modified MoveRegionProcedure to allow null value for destination server i.e. moving a region from specific source server to non-specific/ unknown
destination server (picked by load-balancer) is supported now.
* Removed destination server selection from HMaster.checkIfShouldMoveSystemRegionAsync(), as destination server will be picked by load balancer
Signed-off-by: Michael Stack <st...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f314b591
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f314b591
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f314b591
Branch: refs/heads/master
Commit: f314b5911bf89a7de729b5850df0c6f57a735ce0
Parents: 0339068
Author: Umesh Agashe <ua...@cloudera.com>
Authored: Sat Jul 15 22:51:05 2017 -0700
Committer: Michael Stack <st...@apache.org>
Committed: Tue Aug 8 14:02:11 2017 -0700
----------------------------------------------------------------------
.../src/main/protobuf/MasterProcedure.proto | 3 +-
.../master/assignment/AssignmentManager.java | 44 +++++++++++++++-----
.../master/assignment/MoveRegionProcedure.java | 15 ++++---
3 files changed, 46 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/f314b591/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
----------------------------------------------------------------------
diff --git a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
index 74ae16d..70753c6 100644
--- a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
@@ -387,7 +387,8 @@ enum MoveRegionState {
message MoveRegionStateData {
optional RegionInfo region_info = 1;
required ServerName source_server = 2;
- required ServerName destination_server = 3;
+ // if destination server not specified, its selected with load balancer
+ optional ServerName destination_server = 3;
}
enum GCRegionState {
http://git-wip-us.apache.org/repos/asf/hbase/blob/f314b591/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java
index 255ea5e..54cb1ca 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java
@@ -478,15 +478,13 @@ public class AssignmentManager implements ServerListener {
new Thread(() -> {
try {
synchronized (checkIfShouldMoveSystemRegionLock) {
- List<ServerName> serverList = master.getServerManager()
- .createDestinationServersList(getExcludedServersForSystemTable());
List<RegionPlan> plans = new ArrayList<>();
for (ServerName server : getExcludedServersForSystemTable()) {
List<HRegionInfo> regionsShouldMove = getCarryingSystemTables(server);
if (!regionsShouldMove.isEmpty()) {
for (HRegionInfo regionInfo : regionsShouldMove) {
- RegionPlan plan = new RegionPlan(regionInfo, server,
- getBalancer().randomAssignment(regionInfo, serverList));
+ // null value for dest forces destination server to be selected by balancer
+ RegionPlan plan = new RegionPlan(regionInfo, server, null);
if (regionInfo.isMetaRegion()) {
// Must move meta region first.
moveAsync(plan);
@@ -1648,9 +1646,14 @@ public class AssignmentManager implements ServerListener {
}
// TODO: Optimize balancer. pass a RegionPlan?
- final HashMap<HRegionInfo, ServerName> retainMap = new HashMap<HRegionInfo, ServerName>();
- final List<HRegionInfo> rrList = new ArrayList<HRegionInfo>();
- for (RegionStateNode regionNode: regions.values()) {
+ final HashMap<HRegionInfo, ServerName> retainMap = new HashMap<>();
+ final List<HRegionInfo> userRRList = new ArrayList<>();
+ // regions for system tables requiring reassignment
+ final List<HRegionInfo> sysRRList = new ArrayList<>();
+ for (RegionStateNode regionNode : regions.values()) {
+ boolean sysTable = regionNode.isSystemTable();
+ final List<HRegionInfo> rrList = sysTable ? sysRRList : userRRList;
+
if (regionNode.getRegionLocation() != null) {
retainMap.put(regionNode.getRegionInfo(), regionNode.getRegionLocation());
} else {
@@ -1659,7 +1662,6 @@ public class AssignmentManager implements ServerListener {
}
// TODO: connect with the listener to invalidate the cache
- final LoadBalancer balancer = getBalancer();
// TODO use events
List<ServerName> servers = master.getServerManager().createDestinationServersList();
@@ -1679,13 +1681,35 @@ public class AssignmentManager implements ServerListener {
servers = master.getServerManager().createDestinationServersList();
}
- final boolean isTraceEnabled = LOG.isTraceEnabled();
+ if (!sysRRList.isEmpty()) {
+ // system table regions requiring reassignment are present, get region servers
+ // not available for system table regions
+ final List<ServerName> excludeServers = getExcludedServersForSystemTable();
+ List<ServerName> serversForSysTables = servers.stream()
+ .filter(s -> !excludeServers.contains(s)).collect(Collectors.toList());
+ if (serversForSysTables.isEmpty()) {
+ LOG.warn("No servers available for system table regions, considering all servers!");
+ }
+ LOG.debug("Processing assignment plans for System tables sysServersCount=" +
+ serversForSysTables.size() + ", allServersCount=" + servers.size());
+ processAssignmentPlans(regions, null, sysRRList,
+ serversForSysTables.isEmpty() ? servers : serversForSysTables);
+ }
+
+ processAssignmentPlans(regions, retainMap, userRRList, servers);
+ }
+
+ private void processAssignmentPlans(final HashMap<HRegionInfo, RegionStateNode> regions,
+ final HashMap<HRegionInfo, ServerName> retainMap, final List<HRegionInfo> rrList,
+ final List<ServerName> servers) {
+ boolean isTraceEnabled = LOG.isTraceEnabled();
if (isTraceEnabled) {
LOG.trace("available servers count=" + servers.size() + ": " + servers);
}
+ final LoadBalancer balancer = getBalancer();
// ask the balancer where to place regions
- if (!retainMap.isEmpty()) {
+ if (retainMap != null && !retainMap.isEmpty()) {
if (isTraceEnabled) {
LOG.trace("retain assign regions=" + retainMap);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f314b591/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MoveRegionProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MoveRegionProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MoveRegionProcedure.java
index d8c1b7d..1907e98 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MoveRegionProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MoveRegionProcedure.java
@@ -54,7 +54,6 @@ public class MoveRegionProcedure extends AbstractStateMachineRegionProcedure<Mov
public MoveRegionProcedure(final MasterProcedureEnv env, final RegionPlan plan) {
super(env, plan.getRegionInfo());
- assert plan.getDestination() != null: plan.toString();
this.plan = plan;
}
@@ -70,7 +69,10 @@ public class MoveRegionProcedure extends AbstractStateMachineRegionProcedure<Mov
setNextState(MoveRegionState.MOVE_REGION_ASSIGN);
break;
case MOVE_REGION_ASSIGN:
- addChildProcedure(new AssignProcedure(plan.getRegionInfo(), plan.getDestination()));
+ AssignProcedure assignProcedure = plan.getDestination() == null ?
+ new AssignProcedure(plan.getRegionInfo(), true) :
+ new AssignProcedure(plan.getRegionInfo(), plan.getDestination());
+ addChildProcedure(assignProcedure);
return Flow.NO_MORE_STATE;
default:
throw new UnsupportedOperationException("unhandled state=" + state);
@@ -127,8 +129,10 @@ public class MoveRegionProcedure extends AbstractStateMachineRegionProcedure<Mov
final MoveRegionStateData.Builder state = MoveRegionStateData.newBuilder()
// No need to serialize the HRegionInfo. The super class has the region.
- .setSourceServer(ProtobufUtil.toServerName(plan.getSource()))
- .setDestinationServer(ProtobufUtil.toServerName(plan.getDestination()));
+ .setSourceServer(ProtobufUtil.toServerName(plan.getSource()));
+ if (plan.getDestination() != null) {
+ state.setDestinationServer(ProtobufUtil.toServerName(plan.getDestination()));
+ }
state.build().writeDelimitedTo(stream);
}
@@ -139,7 +143,8 @@ public class MoveRegionProcedure extends AbstractStateMachineRegionProcedure<Mov
final MoveRegionStateData state = MoveRegionStateData.parseDelimitedFrom(stream);
final HRegionInfo regionInfo = getRegion(); // Get it from super class deserialization.
final ServerName sourceServer = ProtobufUtil.toServerName(state.getSourceServer());
- final ServerName destinationServer = ProtobufUtil.toServerName(state.getDestinationServer());
+ final ServerName destinationServer = state.hasDestinationServer() ?
+ ProtobufUtil.toServerName(state.getDestinationServer()) : null;
this.plan = new RegionPlan(regionInfo, sourceServer, destinationServer);
}
}
\ No newline at end of file