You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by vj...@apache.org on 2026/10/12 20:04:54 UTC
(hbase) branch branch-2.6 updated: HBASE-30411 Support regex based RS membership assignment to RS group (#8734)
This is an automated email from the ASF dual-hosted git repository.
virajjasani pushed a commit to branch branch-2.6
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.6 by this push:
new 2f35161d995 HBASE-30411 Support regex based RS membership assignment to RS group (#8734)
2f35161d995 is described below
commit 2f35161d995def16e81401042ec1bc9c7f4b3600
Author: sanjeet006py <36...@users.noreply.github.com>
AuthorDate: Tue Oct 13 01:34:12 2026 +0530
HBASE-30411 Support regex based RS membership assignment to RS group (#8734)
Signed-off-by: Viraj Jasani <vj...@apache.org>
---
.../hbase/rsgroup/RSGroupBasedLoadBalancer.java | 41 +
.../hbase/rsgroup/RSGroupInfoManagerImpl.java | 479 ++++++--
.../hadoop/hbase/rsgroup/TestRSGroupsBase.java | 25 +
.../rsgroup/TestRegexBasedRSGroupMembership.java | 1151 ++++++++++++++++++++
4 files changed, 1620 insertions(+), 76 deletions(-)
diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java
index b33be41f42e..ed3668f10fe 100644
--- a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java
+++ b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java
@@ -78,6 +78,44 @@ public class RSGroupBasedLoadBalancer implements RSGroupableBalancer {
private volatile boolean fallbackEnabled = false;
+ @RestrictedApi(explanation = "Should only be called in tests", link = "",
+ allowedOnPath = ".*/src/test/.*")
+ public void setFallbackEnabledForTest(boolean enabled) {
+ this.fallbackEnabled = enabled;
+ }
+
+ // Test-only call-tracking flags, set true whenever the corresponding assignment method below is
+ // invoked. Reset via resetAssignmentCallFlagsForTest() before the operation under test.
+ private volatile boolean roundRobinAssignmentInvoked = false;
+ private volatile boolean retainAssignmentInvoked = false;
+ private volatile boolean randomAssignmentInvoked = false;
+
+ @RestrictedApi(explanation = "Should only be called in tests", link = "",
+ allowedOnPath = ".*/src/test/.*")
+ public boolean isRoundRobinAssignmentInvoked() {
+ return roundRobinAssignmentInvoked;
+ }
+
+ @RestrictedApi(explanation = "Should only be called in tests", link = "",
+ allowedOnPath = ".*/src/test/.*")
+ public boolean isRetainAssignmentInvoked() {
+ return retainAssignmentInvoked;
+ }
+
+ @RestrictedApi(explanation = "Should only be called in tests", link = "",
+ allowedOnPath = ".*/src/test/.*")
+ public boolean isRandomAssignmentInvoked() {
+ return randomAssignmentInvoked;
+ }
+
+ @RestrictedApi(explanation = "Should only be called in tests", link = "",
+ allowedOnPath = ".*/src/test/.*")
+ public void resetAssignmentCallFlagsForTest() {
+ roundRobinAssignmentInvoked = false;
+ retainAssignmentInvoked = false;
+ randomAssignmentInvoked = false;
+ }
+
/**
* Used by reflection in {@link org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory}.
*/
@@ -161,6 +199,7 @@ public class RSGroupBasedLoadBalancer implements RSGroupableBalancer {
@NonNull
public Map<ServerName, List<RegionInfo>> roundRobinAssignment(List<RegionInfo> regions,
List<ServerName> servers) throws HBaseIOException {
+ roundRobinAssignmentInvoked = true;
Map<ServerName, List<RegionInfo>> assignments = Maps.newHashMap();
List<Pair<List<RegionInfo>, List<ServerName>>> pairs =
generateGroupAssignments(regions, servers);
@@ -177,6 +216,7 @@ public class RSGroupBasedLoadBalancer implements RSGroupableBalancer {
@NonNull
public Map<ServerName, List<RegionInfo>> retainAssignment(Map<RegionInfo, ServerName> regions,
List<ServerName> servers) throws HBaseIOException {
+ retainAssignmentInvoked = true;
try {
Map<ServerName, List<RegionInfo>> assignments = new TreeMap<>();
List<Pair<List<RegionInfo>, List<ServerName>>> pairs =
@@ -199,6 +239,7 @@ public class RSGroupBasedLoadBalancer implements RSGroupableBalancer {
@Override
public ServerName randomAssignment(RegionInfo region, List<ServerName> servers)
throws HBaseIOException {
+ randomAssignmentInvoked = true;
List<Pair<List<RegionInfo>, List<ServerName>>> pairs =
generateGroupAssignments(Lists.newArrayList(region), servers);
List<ServerName> filteredServers = pairs.iterator().next().getSecond();
diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java
index 79d6ffdd605..ce4c61f73e6 100644
--- a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java
+++ b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java
@@ -21,6 +21,7 @@ import com.google.protobuf.ServiceException;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -32,6 +33,9 @@ import java.util.OptionalLong;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
+import java.util.regex.Pattern;
+import java.util.regex.PatternSyntaxException;
+import java.util.stream.Collectors;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Coprocessor;
import org.apache.hadoop.hbase.DoNotRetryIOException;
@@ -81,12 +85,29 @@ import org.slf4j.LoggerFactory;
import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
import org.apache.hbase.thirdparty.com.google.common.collect.Maps;
-import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
/**
* This is an implementation of {@link RSGroupInfoManager} which makes use of an HBase table as the
* persistence store for the group information. It also makes use of zookeeper to store group
* information needed for bootstrapping during offline mode.
+ * <h2>Admin-managed and auto-managed groups</h2> An <b>admin-managed</b> group is a non-default
+ * group that is not regex-governed (see {@code hbase.rsgroup.regex.<group>}). Its servers are set
+ * by an administrator and are stored in hbase:rsgroup and ZooKeeper. A group whose regex is invalid
+ * is admin-managed until the regex is fixed. In that time the servers that the group holds in
+ * memory count as admin-managed when the membership is computed
+ * ({@code resolveRegexBasedRSGroupMembership}).
+ * <p/>
+ * An <b>auto-managed</b> group is 'default' or a regex-governed group. Its servers are computed
+ * from the live servers. In HBase 2.x, the computed servers are also stored in hbase:rsgroup and
+ * ZooKeeper on a best-effort basis, because table readers such as {@code RSGroupTableAccessor} read
+ * the membership directly from the table.
+ * <p/>
+ * Auto-managed groups exist to make RSGroup work with auto-scaling when HBase runs in a cloud
+ * environment such as AWS or GCP. An admin-managed group does not give the elasticity that
+ * automatic upscaling and downscaling need: each new RegionServer must be moved into the group by
+ * hand. It fits a fixed number of RegionServers, for example to host internal system tables like
+ * hbase:meta and hbase:namespace. End user facing tables must be placed on an elastic group, so
+ * RegionServers join and leave it automatically. That is the need for auto-managed groups.
* <h2>Concurrency</h2> RSGroup state is kept locally in Maps. There is a rsgroup name to cached
* RSGroupInfo Map at {@link #rsGroupMap} and a Map of tables to the name of the rsgroup they belong
* too (in {@link #tableMap}). These Maps are persisted to the hbase:rsgroup table (and cached in
@@ -175,6 +196,141 @@ final class RSGroupInfoManagerImpl implements RSGroupInfoManager {
}
}
+ static final String RS_GROUP_REGEX_PREFIX = "hbase.rsgroup.regex.";
+ private static final Pattern GROUP_NAME_PATTERN = Pattern.compile("[a-zA-Z0-9_]+");
+
+ static Map<String, Pattern> getRegexGroupMap(Configuration conf) {
+ Map<String, String> rsGroupNameToRegexMap = conf.getPropsWithPrefix(RS_GROUP_REGEX_PREFIX);
+ Map<String, Pattern> rsGroupNameToPatternMap = new HashMap<>();
+ for (Map.Entry<String, String> e : rsGroupNameToRegexMap.entrySet()) {
+ if (e.getValue() == null) {
+ // getPropsWithPrefix reads names and values separately, so an entry removed by a concurrent
+ // configuration reload can show up here with a null value. Treat it as unset.
+ continue;
+ }
+ if (!GROUP_NAME_PATTERN.matcher(e.getKey()).matches()) {
+ LOG.warn("Ignoring {}{} -- '{}' is not a valid RSGroup name (it must match {})",
+ RS_GROUP_REGEX_PREFIX, e.getKey(), e.getKey(), GROUP_NAME_PATTERN.pattern());
+ continue;
+ }
+ if (RSGroupInfo.DEFAULT_GROUP.equals(e.getKey())) {
+ LOG.warn("Ignoring {}{} -- regex-based membership cannot target the reserved '{}' group",
+ RS_GROUP_REGEX_PREFIX, e.getKey(), RSGroupInfo.DEFAULT_GROUP);
+ continue;
+ }
+ try {
+ rsGroupNameToPatternMap.put(e.getKey(), Pattern.compile(e.getValue()));
+ } catch (PatternSyntaxException ex) {
+ LOG.warn("Invalid regex '{}' for RSGroup '{}' ({}{}); ignoring this entry", e.getValue(),
+ e.getKey(), RS_GROUP_REGEX_PREFIX, e.getKey());
+ }
+ }
+ LOG.debug("Resolved regex-based RSGroup membership config: {}", rsGroupNameToPatternMap);
+ return rsGroupNameToPatternMap;
+ }
+
+ private static List<String> getMatchingRSGroupNames(String hostname,
+ Map<String, Pattern> rsGroupNameToPatternMap) {
+ List<String> rsGroupNames = new ArrayList<>();
+ for (Map.Entry<String, Pattern> e : rsGroupNameToPatternMap.entrySet()) {
+ if (e.getValue().matcher(hostname).matches()) {
+ rsGroupNames.add(e.getKey());
+ }
+ }
+ return rsGroupNames;
+ }
+
+ static Map<Address, String> resolveServerAddrToRSGroupName(Set<Address> onlineServers,
+ Map<String, Pattern> rsGroupNameToPatternMap, Set<String> existingGroupNames) {
+ Map<Address, String> serverAddrToRSGroupName = new HashMap<>();
+ if (rsGroupNameToPatternMap.isEmpty()) {
+ return serverAddrToRSGroupName;
+ }
+ Set<String> nonExistingRSGroupNames = new HashSet<>();
+ for (Address server : onlineServers) {
+ List<String> matchingRSGroupNames =
+ getMatchingRSGroupNames(server.getHostName(), rsGroupNameToPatternMap);
+ if (matchingRSGroupNames.isEmpty()) {
+ continue;
+ }
+ if (matchingRSGroupNames.size() > 1) {
+ LOG.warn("Server {} hostname matches regexes for multiple RSGroups {}; treating it as "
+ + "unmatched by regex (falls back to default) until the overlapping "
+ + "hbase.rsgroup.regex.* entries are fixed", server, matchingRSGroupNames);
+ continue;
+ }
+ String rsGroupName = matchingRSGroupNames.get(0);
+ if (existingGroupNames.contains(rsGroupName)) {
+ serverAddrToRSGroupName.put(server, rsGroupName);
+ } else if (nonExistingRSGroupNames.add(rsGroupName)) {
+ LOG.warn(
+ "Config {}{} matches server {} but RSGroup '{}' does not exist -- create it with "
+ + "addRSGroup first; treating this server as unmatched by regex until then",
+ RS_GROUP_REGEX_PREFIX, rsGroupName, server, rsGroupName);
+ }
+ }
+ LOG.debug("Resolved server address to RSGroup name map: {}", serverAddrToRSGroupName);
+ return serverAddrToRSGroupName;
+ }
+
+ static boolean wouldEmptyDefaultGroup(Set<Address> onlineServers,
+ Set<Address> adminManagedServers, Set<Address> regexMatchedServers) {
+ for (Address server : onlineServers) {
+ if (!adminManagedServers.contains(server) && !regexMatchedServers.contains(server)) {
+ return false;
+ }
+ }
+ return !onlineServers.isEmpty();
+ }
+
+ static final class RegexBasedRSGroupMembershipResolution {
+ final Set<Address> adminManagedServers;
+ final Map<Address, String> regexMatchedServers;
+
+ RegexBasedRSGroupMembershipResolution(Set<Address> adminManagedServers,
+ Map<Address, String> regexMatchedServers) {
+ this.adminManagedServers = adminManagedServers;
+ this.regexMatchedServers = regexMatchedServers;
+ }
+ }
+
+ static RegexBasedRSGroupMembershipResolution resolveRegexBasedRSGroupMembership(
+ Map<String, Pattern> rsGroupNameToPatternMap, Set<Address> onlineServers,
+ Collection<RSGroupInfo> existingGroups) {
+ Set<String> existingGroupNames =
+ existingGroups.stream().map(RSGroupInfo::getName).collect(Collectors.toSet());
+ Set<Address> adminManagedServers = new HashSet<>();
+ for (RSGroupInfo existingGroup : existingGroups) {
+ if (
+ !RSGroupInfo.DEFAULT_GROUP.equals(existingGroup.getName())
+ && !rsGroupNameToPatternMap.containsKey(existingGroup.getName())
+ ) {
+ adminManagedServers.addAll(existingGroup.getServers());
+ }
+ }
+ Map<Address, String> regexMatchedServers =
+ resolveServerAddrToRSGroupName(onlineServers, rsGroupNameToPatternMap, existingGroupNames);
+ // A server stored in an admin-managed group that a regex now claims belongs to the regex group.
+ adminManagedServers.removeAll(regexMatchedServers.keySet());
+ if (wouldEmptyDefaultGroup(onlineServers, adminManagedServers, regexMatchedServers.keySet())) {
+ // Regex placement is still applied: every server lands in its correct group. Operators are
+ // expected to configure regexes carefully, so this is a warning, not an error.
+ LOG.warn(
+ "Regex-based RSGroup membership leaves RSGroup '{}' with no online servers -- every "
+ + "online server is either admin-managed or matches a {}* entry. Tables in RSGroup '{}' "
+ + "(including system tables, unless they are in another RSGroup) have no server of "
+ + "their own to be assigned to; set {} to let them fall back to servers of other "
+ + "RSGroups",
+ RSGroupInfo.DEFAULT_GROUP, RS_GROUP_REGEX_PREFIX, RSGroupInfo.DEFAULT_GROUP,
+ RSGroupBasedLoadBalancer.FALLBACK_GROUP_ENABLE_KEY);
+ }
+ LOG.debug(
+ "Regex-based RSGroup membership resolution: onlineServers={}, "
+ + "adminManagedServers={}, regexMatchedServers={}",
+ onlineServers, adminManagedServers, regexMatchedServers);
+ return new RegexBasedRSGroupMembershipResolution(adminManagedServers, regexMatchedServers);
+ }
+
private RSGroupInfoManagerImpl(MasterServices masterServices) throws IOException {
this.masterServices = masterServices;
this.watcher = masterServices.getZooKeeper();
@@ -241,8 +397,11 @@ final class RSGroupInfoManagerImpl implements RSGroupInfoManager {
@Override
public synchronized Set<Address> moveServers(Set<Address> servers, String srcGroup,
String dstGroup) throws IOException {
- RSGroupInfo src = getRSGroupInfo(srcGroup);
- RSGroupInfo dst = getRSGroupInfo(dstGroup);
+ // Mutate copies, not the live rsGroupMap entries -- flushConfig() below can still reject
+ // this change (e.g. the regex-membership invariant), and the live state must stay untouched
+ // until the change is actually persisted.
+ RSGroupInfo src = new RSGroupInfo(getRSGroupInfo(srcGroup));
+ RSGroupInfo dst = new RSGroupInfo(getRSGroupInfo(dstGroup));
Set<Address> movedServers = new HashSet<>();
// If destination is 'default' rsgroup, only add servers that are online. If not online, drop
// it. If not 'default' group, add server to 'dst' rsgroup EVEN IF IT IS NOT online (could be a
@@ -263,6 +422,7 @@ final class RSGroupInfoManagerImpl implements RSGroupInfoManager {
dst.addServer(el);
movedServers.add(el);
}
+ checkMoveAgreesWithRegex(movedServers, dst.getName());
Map<String, RSGroupInfo> newGroupMap = Maps.newHashMap(rsGroupMap);
newGroupMap.put(src.getName(), src);
newGroupMap.put(dst.getName(), dst);
@@ -270,6 +430,37 @@ final class RSGroupInfoManagerImpl implements RSGroupInfoManager {
return movedServers;
}
+ /**
+ * Regex-based membership always wins: the next server event or refresh puts a server back where
+ * its hostname says, so an explicit move that disagrees would only make regions flap. Such a move
+ * is rejected. A server whose hostname matches no regex (or an ambiguous or dangling one, which
+ * fall back to ordinary placement) may go anywhere except into a regex-governed group.
+ */
+ private void checkMoveAgreesWithRegex(Set<Address> movedServers, String dstGroup)
+ throws ConstraintException {
+ Map<String, Pattern> regexGroupMap = getRegexGroupMap(masterServices.getConfiguration());
+ if (regexGroupMap.isEmpty() || movedServers.isEmpty()) {
+ return;
+ }
+ Map<Address, String> expectedGroups =
+ resolveServerAddrToRSGroupName(movedServers, regexGroupMap, rsGroupMap.keySet());
+ for (Address server : movedServers) {
+ String expectedGroup = expectedGroups.get(server);
+ boolean isWrongMove = expectedGroup != null
+ ? !expectedGroup.equals(dstGroup)
+ : regexGroupMap.containsKey(dstGroup);
+ if (isWrongMove) {
+ String reason = expectedGroup != null
+ ? "its membership is decided by " + RS_GROUP_REGEX_PREFIX + expectedGroup
+ + ", under which its hostname belongs in '" + expectedGroup + "'"
+ : "its hostname does not match " + RS_GROUP_REGEX_PREFIX + dstGroup
+ + ", which decides the membership of that RSGroup";
+ throw new ConstraintException("Server " + server + " cannot be moved to RSGroup '"
+ + dstGroup + "': " + reason + ". Change that config instead.");
+ }
+ }
+ }
+
@Override
public RSGroupInfo getRSGroupOfServer(Address serverHostPort) throws IOException {
for (RSGroupInfo info : rsGroupMap.values()) {
@@ -347,9 +538,12 @@ final class RSGroupInfoManagerImpl implements RSGroupInfoManager {
@Override
public void moveServersAndTables(Set<Address> servers, Set<TableName> tables, String srcGroup,
String dstGroup) throws IOException {
- // get server's group
- RSGroupInfo srcGroupInfo = getRSGroupInfo(srcGroup);
- RSGroupInfo dstGroupInfo = getRSGroupInfo(dstGroup);
+ // get server's group; mutate copies, not the live rsGroupMap entries -- flushConfig() below
+ // can still reject this change, and the live state must stay untouched until persisted.
+ RSGroupInfo srcGroupInfo = new RSGroupInfo(getRSGroupInfo(srcGroup));
+ RSGroupInfo dstGroupInfo = new RSGroupInfo(getRSGroupInfo(dstGroup));
+
+ checkMoveAgreesWithRegex(servers, dstGroup);
// move servers
for (Address el : servers) {
@@ -377,8 +571,11 @@ final class RSGroupInfoManagerImpl implements RSGroupInfoManager {
if (rsGroupInfo != null) {
RSGroupInfo newRsGroupInfo = rsGroupInfos.get(rsGroupInfo.getName());
if (newRsGroupInfo == null) {
- rsGroupInfo.removeServer(el);
- rsGroupInfos.put(rsGroupInfo.getName(), rsGroupInfo);
+ // Mutate a copy, not the live rsGroupMap entry -- flushConfig() below can still
+ // reject this change, and the live state must stay untouched until persisted.
+ newRsGroupInfo = new RSGroupInfo(rsGroupInfo);
+ newRsGroupInfo.removeServer(el);
+ rsGroupInfos.put(newRsGroupInfo.getName(), newRsGroupInfo);
} else {
newRsGroupInfo.removeServer(el);
rsGroupInfos.put(newRsGroupInfo.getName(), newRsGroupInfo);
@@ -464,11 +661,14 @@ final class RSGroupInfoManagerImpl implements RSGroupInfoManager {
throw new ConstraintException(
"configuration of " + RSGroupInfo.DEFAULT_GROUP + " can't be stored persistently");
}
- RSGroupInfo rsGroupInfo = getRSGroupInfo(groupName);
- new HashSet<>(rsGroupInfo.getConfiguration().keySet())
+ RSGroupInfo rsGroupInfo = new RSGroupInfo(getRSGroupInfo(groupName));
+ // Iterate over a snapshot of the keys, as removeConfiguration modifies the backing map.
+ new ArrayList<>(rsGroupInfo.getConfiguration().keySet())
.forEach(rsGroupInfo::removeConfiguration);
configuration.forEach(rsGroupInfo::setConfiguration);
- flushConfig();
+ Map<String, RSGroupInfo> newGroupMap = Maps.newHashMap(rsGroupMap);
+ newGroupMap.put(groupName, rsGroupInfo);
+ flushConfig(newGroupMap);
}
List<RSGroupInfo> retrieveGroupListFromGroupTable() throws IOException {
@@ -526,6 +726,8 @@ final class RSGroupInfoManagerImpl implements RSGroupInfoManager {
* startup of the manager.
*/
private synchronized void refresh(boolean forceOnline) throws IOException {
+ LOG.debug("Refreshing RSGroup info from source of truth: forceOnline={}, isOnline={}",
+ forceOnline, isOnline());
List<RSGroupInfo> groupList = new LinkedList<>();
// Overwrite anything read from zk, group table is source of truth
@@ -549,22 +751,42 @@ final class RSGroupInfoManagerImpl implements RSGroupInfoManager {
}
}
- // This is added to the last of the list so it overwrites the 'default' rsgroup loaded
- // from region group table or zk
- groupList
- .add(new RSGroupInfo(RSGroupInfo.DEFAULT_GROUP, getDefaultServers(groupList), orphanTables));
+ // Replace the 'default' rsgroup loaded from the group table or zk with a freshly built
+ // one -- its server membership is always recomputed, never trusted from storage.
+ groupList.removeIf(group -> group.getName().equals(RSGroupInfo.DEFAULT_GROUP));
+ groupList.add(new RSGroupInfo(RSGroupInfo.DEFAULT_GROUP, new TreeSet<Address>(), orphanTables));
// populate the data
HashMap<String, RSGroupInfo> newGroupMap = Maps.newHashMap();
- HashMap<TableName, String> newTableMap = Maps.newHashMap();
for (RSGroupInfo group : groupList) {
newGroupMap.put(group.getName(), group);
+ }
+ // Server membership for 'default' and for every regex-governed group is always recomputed
+ // on the fly -- the stored copy is never trusted, it is only a snapshot for direct readers of
+ // hbase:rsgroup.
+ Map<String, SortedSet<Address>> autoManagedServers =
+ computeAutoManagedRSGroupServers(groupList);
+ for (Map.Entry<String, SortedSet<Address>> entry : autoManagedServers.entrySet()) {
+ RSGroupInfo stored = newGroupMap.get(entry.getKey());
+ if (!RSGroupInfo.DEFAULT_GROUP.equals(entry.getKey()) && !stored.getServers().isEmpty()) {
+ LOG.info(
+ "Ignoring servers {} stored for regex-governed RSGroup '{}'; its membership is "
+ + "computed on the fly and the stored copy is replaced on the next flush",
+ stored.getServers(), entry.getKey());
+ }
+ }
+ applyAutoManagedRSGroupServers(newGroupMap, autoManagedServers);
+ dropRegexGovernedServersFromOtherGroups(newGroupMap, autoManagedServers);
+
+ HashMap<TableName, String> newTableMap = Maps.newHashMap();
+ for (RSGroupInfo group : newGroupMap.values()) {
for (TableName table : group.getTables()) {
newTableMap.put(table, group.getName());
}
}
resetRSGroupAndTableMaps(newGroupMap, newTableMap);
updateCacheOfRSGroups(rsGroupMap.keySet());
+ LOG.info("Refresh completed successfully");
}
private synchronized Map<TableName, String> flushConfigTable(Map<String, RSGroupInfo> groupMap)
@@ -616,13 +838,16 @@ final class RSGroupInfoManagerImpl implements RSGroupInfoManager {
Map<String, RSGroupInfo> oldGroupMap = Maps.newHashMap(rsGroupMap);
RSGroupInfo oldDefaultGroup = oldGroupMap.remove(RSGroupInfo.DEFAULT_GROUP);
RSGroupInfo newDefaultGroup = newGroupMap.remove(RSGroupInfo.DEFAULT_GROUP);
+ // Servers of regex-governed groups are computed on the fly, like those of the default
+ // group, so their changes are allowed here too.
if (
- !oldGroupMap.equals(newGroupMap)
+ !withoutRegexGovernedServers(oldGroupMap).equals(withoutRegexGovernedServers(newGroupMap))
/* compare both tables and servers in other groups */ || !oldDefaultGroup.getTables()
.equals(newDefaultGroup.getTables())
/* compare tables in default group */
) {
- throw new IOException("Only servers in default group can be updated during offline mode");
+ throw new IOException("Only servers in the default group and in regex-governed groups can "
+ + "be updated during offline mode");
}
// Restore newGroupMap by putting its default group back
@@ -633,8 +858,8 @@ final class RSGroupInfoManagerImpl implements RSGroupInfoManager {
rsGroupMap = newGroupMap;
// Do not need to update tableMap
- // because only the update on servers in default group is allowed above,
- // or IOException will be thrown
+ // because only server-set updates are allowed above,
+ // or an IOException will be thrown
return;
}
@@ -676,6 +901,27 @@ final class RSGroupInfoManagerImpl implements RSGroupInfoManager {
updateCacheOfRSGroups(newGroupMap.keySet());
}
+ /**
+ * Returns a copy of {@code groupMap} where every regex-governed group has an empty server set.
+ * Used to tell whether two group maps differ in anything other than the membership that is
+ * recomputed from the live servers (see {@link #refresh(boolean)}).
+ */
+ private Map<String, RSGroupInfo> withoutRegexGovernedServers(Map<String, RSGroupInfo> groupMap) {
+ Set<String> regexGovernedGroups = getRegexGroupMap(masterServices.getConfiguration()).keySet();
+ Map<String, RSGroupInfo> result = Maps.newHashMap(groupMap);
+ for (String groupName : regexGovernedGroups) {
+ RSGroupInfo info = groupMap.get(groupName);
+ if (info == null || info.getServers().isEmpty()) {
+ continue;
+ }
+ RSGroupInfo stripped = new RSGroupInfo(groupName);
+ stripped.addAllTables(info.getTables());
+ info.getConfiguration().forEach(stripped::setConfiguration);
+ result.put(groupName, stripped);
+ }
+ return result;
+ }
+
/**
* Make changes visible. Caller must be synchronized on 'this'.
*/
@@ -695,69 +941,149 @@ final class RSGroupInfoManagerImpl implements RSGroupInfoManager {
this.prevRSGroups.addAll(currentGroups);
}
- // Called by getDefaultServers. Presume it has lock in place.
- private List<ServerName> getOnlineRS() throws IOException {
- if (masterServices != null) {
- return masterServices.getServerManager().getOnlineServersList();
+ private Map<String, SortedSet<Address>>
+ computeAutoManagedRSGroupServers(Collection<RSGroupInfo> existingGroups) {
+ Set<Address> onlineServers = getOnlineServers(masterServices);
+ Map<String, Pattern> rsGroupNameToPatternMap =
+ getRegexGroupMap(masterServices.getConfiguration());
+ RegexBasedRSGroupMembershipResolution resolution =
+ resolveRegexBasedRSGroupMembership(rsGroupNameToPatternMap, onlineServers, existingGroups);
+
+ Set<String> existingGroupNames =
+ existingGroups.stream().map(RSGroupInfo::getName).collect(Collectors.toSet());
+ Map<String, SortedSet<Address>> result = new HashMap<>();
+ for (String rsGroupName : rsGroupNameToPatternMap.keySet()) {
+ if (existingGroupNames.contains(rsGroupName)) {
+ result.put(rsGroupName, new TreeSet<>());
+ }
}
- LOG.debug("Reading online RS from zookeeper");
- List<ServerName> servers = new LinkedList<>();
- try {
- for (String el : ZKUtil.listChildrenNoWatch(watcher, watcher.getZNodePaths().rsZNode)) {
- servers.add(ServerName.parseServerName(el));
+ result.put(RSGroupInfo.DEFAULT_GROUP, new TreeSet<>());
+
+ for (Address server : onlineServers) {
+ String regexGroupName = resolution.regexMatchedServers.get(server);
+ if (regexGroupName != null) {
+ result.get(regexGroupName).add(server);
+ continue;
}
- } catch (KeeperException e) {
- throw new IOException("Failed to retrieve server list from zookeeper", e);
+ if (resolution.adminManagedServers.contains(server)) {
+ continue;
+ }
+ result.get(RSGroupInfo.DEFAULT_GROUP).add(server);
}
- return servers;
+ for (Map.Entry<String, SortedSet<Address>> entry : result.entrySet()) {
+ if (entry.getValue().isEmpty() && !RSGroupInfo.DEFAULT_GROUP.equals(entry.getKey())) {
+ LOG.warn(
+ "Regex-governed RSGroup '{}' has no online servers; any table assigned to it has "
+ + "nowhere to go (check {}{} and the online servers' hostnames)",
+ entry.getKey(), RS_GROUP_REGEX_PREFIX, entry.getKey());
+ }
+ }
+ LOG.debug("Computed auto-managed RSGroup server membership: {}", result);
+ return result;
}
- // Called by ServerEventsListenerThread. Presume it has lock on this manager when it runs.
- private SortedSet<Address> getDefaultServers() throws IOException {
- return getDefaultServers(listRSGroups());
+ private static void applyAutoManagedRSGroupServers(Map<String, RSGroupInfo> groupMap,
+ Map<String, SortedSet<Address>> newRSGroupToServers) {
+ for (Map.Entry<String, SortedSet<Address>> entry : newRSGroupToServers.entrySet()) {
+ RSGroupInfo oldInfo = groupMap.get(entry.getKey());
+ if (oldInfo == null) {
+ continue;
+ }
+ RSGroupInfo newInfo = new RSGroupInfo(entry.getKey(), entry.getValue());
+ newInfo.addAllTables(oldInfo.getTables());
+ oldInfo.getConfiguration().forEach(newInfo::setConfiguration);
+ groupMap.put(entry.getKey(), newInfo);
+ }
}
- // Called by ServerEventsListenerThread. Presume it has lock on this manager when it runs.
- private SortedSet<Address> getDefaultServers(List<RSGroupInfo> rsGroupInfoList)
- throws IOException {
- // Build a list of servers in other groups than default group, from rsGroupMap
- Set<Address> serversInOtherGroup = new HashSet<>();
- for (RSGroupInfo group : rsGroupInfoList) {
- if (!RSGroupInfo.DEFAULT_GROUP.equals(group.getName())) { // not default group
- serversInOtherGroup.addAll(group.getServers());
+ /**
+ * Stored membership of admin-managed groups is not recomputed on refresh, so a server that was
+ * placed in one of them before a regex claimed it would end up in two groups. Unlike the runtime
+ * paths, refresh cannot reject this: it runs in the startup worker, which retries forever on
+ * failure and would keep the manager offline. Instead, resolve it the way
+ * {@link #computeAutoManagedRSGroupServers} already does -- the regex wins -- in memory only;
+ * storage is corrected by the next flush.
+ */
+ private static void dropRegexGovernedServersFromOtherGroups(Map<String, RSGroupInfo> groupMap,
+ Map<String, SortedSet<Address>> autoManagedServers) {
+ Map<Address, String> regexGovernedServers = new HashMap<>();
+ for (Map.Entry<String, SortedSet<Address>> entry : autoManagedServers.entrySet()) {
+ if (!RSGroupInfo.DEFAULT_GROUP.equals(entry.getKey())) {
+ entry.getValue().forEach(server -> regexGovernedServers.put(server, entry.getKey()));
}
}
-
- // Get all online servers from Zookeeper and find out servers in default group
- SortedSet<Address> defaultServers = Sets.newTreeSet();
- for (ServerName serverName : getOnlineRS()) {
- Address server = Address.fromParts(serverName.getHostname(), serverName.getPort());
- if (!serversInOtherGroup.contains(server)) { // not in other groups
- defaultServers.add(server);
+ if (regexGovernedServers.isEmpty()) {
+ return;
+ }
+ for (Map.Entry<String, RSGroupInfo> entry : groupMap.entrySet()) {
+ RSGroupInfo info = entry.getValue();
+ if (autoManagedServers.containsKey(info.getName())) {
+ continue;
+ }
+ Set<Address> overlap = new TreeSet<>();
+ for (Address server : info.getServers()) {
+ if (regexGovernedServers.containsKey(server)) {
+ overlap.add(server);
+ }
+ }
+ if (overlap.isEmpty()) {
+ continue;
}
+ LOG.warn(
+ "Servers {} are stored in RSGroup '{}' but match a {}* entry; placing them in the "
+ + "regex-governed RSGroup instead and dropping the stored copy",
+ overlap, info.getName(), RS_GROUP_REGEX_PREFIX);
+ SortedSet<Address> kept = new TreeSet<>(info.getServers());
+ kept.removeAll(overlap);
+ RSGroupInfo newInfo = new RSGroupInfo(info.getName(), kept);
+ newInfo.addAllTables(info.getTables());
+ info.getConfiguration().forEach(newInfo::setConfiguration);
+ entry.setValue(newInfo);
}
- return defaultServers;
}
- // Called by ServerEventsListenerThread. Synchronize on this because redoing
- // the rsGroupMap then writing it out.
- private synchronized void updateDefaultServers(SortedSet<Address> servers) throws IOException {
- RSGroupInfo info = rsGroupMap.get(RSGroupInfo.DEFAULT_GROUP);
- RSGroupInfo newInfo = new RSGroupInfo(info.getName(), servers, info.getTables());
- HashMap<String, RSGroupInfo> newGroupMap = Maps.newHashMap(rsGroupMap);
- newGroupMap.put(newInfo.getName(), newInfo);
- flushConfig(newGroupMap);
+ /**
+ * Recomputes server membership of 'default' and of every regex-governed group from the live
+ * servers and flushes the result. If the flush fails (for example because {@code hbase:rsgroup}
+ * is disabled) the new membership is still applied in memory, so membership stays visible, and
+ * the exception is rethrown. Storage catches up on the next successful flush.
+ */
+ private synchronized void updateAutoManagedRSGroupServers() throws IOException {
+ Map<String, RSGroupInfo> currentGroups = rsGroupMap;
+ Map<String, SortedSet<Address>> autoManagedServers =
+ computeAutoManagedRSGroupServers(currentGroups.values());
+ Map<String, RSGroupInfo> newGroupMap = Maps.newHashMap(currentGroups);
+ applyAutoManagedRSGroupServers(newGroupMap, autoManagedServers);
+ dropRegexGovernedServersFromOtherGroups(newGroupMap, autoManagedServers);
+ boolean changed = false;
+ for (Map.Entry<String, RSGroupInfo> entry : newGroupMap.entrySet()) {
+ if (!entry.getValue().getServers().equals(currentGroups.get(entry.getKey()).getServers())) {
+ changed = true;
+ break;
+ }
+ }
+ if (!changed) {
+ return;
+ }
+ try {
+ flushConfig(newGroupMap);
+ } catch (IOException e) {
+ resetRSGroupAndTableMaps(newGroupMap, tableMap);
+ throw e;
+ }
+ LOG.info("Updated auto-managed RSGroup servers, {} servers",
+ autoManagedServers.values().stream().mapToInt(SortedSet::size).sum());
}
/**
- * Calls {@link RSGroupInfoManagerImpl#updateDefaultServers(SortedSet)} to update list of known
+ * Calls {@link RSGroupInfoManagerImpl#updateAutoManagedRSGroupServers()} to update list of known
* servers. Notifications about server changes are received by registering {@link ServerListener}.
* As a listener, we need to return immediately, so the real work of updating the servers is done
* asynchronously in this thread.
*/
private class ServerEventsListenerThread extends Thread implements ServerListener {
private final Logger LOG = LoggerFactory.getLogger(ServerEventsListenerThread.class);
- private boolean changed = false;
+ private int eventCount = 0;
ServerEventsListenerThread() {
setDaemon(true);
@@ -765,44 +1091,44 @@ final class RSGroupInfoManagerImpl implements RSGroupInfoManager {
@Override
public void serverAdded(ServerName serverName) {
+ LOG.debug("Server added: {}", serverName);
serverChanged();
}
@Override
public void serverRemoved(ServerName serverName) {
+ LOG.debug("Server removed: {}", serverName);
serverChanged();
}
private synchronized void serverChanged() {
- changed = true;
+ eventCount++;
this.notify();
}
@Override
public void run() {
setName(ServerEventsListenerThread.class.getName() + "-" + masterServices.getServerName());
- SortedSet<Address> prevDefaultServers = new TreeSet<>();
while (isMasterRunning(masterServices)) {
try {
- LOG.info("Updating default servers.");
- SortedSet<Address> servers = RSGroupInfoManagerImpl.this.getDefaultServers();
- if (!servers.equals(prevDefaultServers)) {
- RSGroupInfoManagerImpl.this.updateDefaultServers(servers);
- prevDefaultServers = servers;
- LOG.info("Updated with servers: " + servers.size());
- }
try {
synchronized (this) {
- while (!changed) {
+ while (eventCount <= 0) {
wait();
}
- changed = false;
}
} catch (InterruptedException e) {
LOG.warn("Interrupted", e);
+ continue;
+ }
+ RSGroupInfoManagerImpl.this.updateAutoManagedRSGroupServers();
+ synchronized (this) {
+ if (eventCount > 0) {
+ eventCount--;
+ }
}
- } catch (IOException e) {
- LOG.warn("Failed to update default servers", e);
+ } catch (Exception e) {
+ LOG.warn("Failed to update auto-managed RSGroup servers", e);
}
}
}
@@ -928,8 +1254,9 @@ final class RSGroupInfoManagerImpl implements RSGroupInfoManager {
}
private void checkGroupName(String groupName) throws ConstraintException {
- if (!groupName.matches("[a-zA-Z0-9_]+")) {
- throw new ConstraintException("RSGroup name should only contain alphanumeric characters");
+ if (!GROUP_NAME_PATTERN.matcher(groupName).matches()) {
+ throw new ConstraintException(
+ "RSGroup name '" + groupName + "' must match " + GROUP_NAME_PATTERN.pattern());
}
}
}
diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java
index 54946f303bb..fc3eb0f43e7 100644
--- a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java
+++ b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hbase.rsgroup;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.IOException;
@@ -203,6 +205,12 @@ public abstract class TestRSGroupsBase extends AbstractTestUpdateConfiguration {
}
rsGroupAdmin.moveServers(set, groupName);
RSGroupInfo result = rsGroupAdmin.getRSGroupInfo(groupName);
+ assertEquals(set, result.getServers());
+ Set<Address> remainingDefaultServers =
+ rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers();
+ for (Address server : set) {
+ assertFalse(remainingDefaultServers.contains(server));
+ }
return result;
}
@@ -211,6 +219,23 @@ public abstract class TestRSGroupsBase extends AbstractTestUpdateConfiguration {
rsGroupAdmin.moveTables(groupInfo.getTables(), RSGroupInfo.DEFAULT_GROUP);
rsGroupAdmin.moveServers(groupInfo.getServers(), RSGroupInfo.DEFAULT_GROUP);
rsGroupAdmin.removeRSGroup(groupName);
+ assertFalse(rsGroupAdmin.listRSGroups().stream().anyMatch(g -> g.getName().equals(groupName)));
+ Set<Address> onlineServers = new HashSet<>();
+ for (ServerName sn : master.getServerManager().getOnlineServersList()) {
+ onlineServers.add(sn.getAddress());
+ }
+ RSGroupInfo defaultInfo = rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP);
+ for (Address server : groupInfo.getServers()) {
+ // moveServers()-to-'default' silently drops servers that are no longer online (see
+ // RSGroupInfoManagerImpl#moveServers), so only online servers are guaranteed to land
+ // in 'default'; an offline server's former group membership is simply dropped.
+ if (onlineServers.contains(server)) {
+ assertTrue(defaultInfo.getServers().contains(server));
+ }
+ }
+ for (TableName table : groupInfo.getTables()) {
+ assertTrue(defaultInfo.getTables().contains(table));
+ }
}
protected void deleteTableIfNecessary() throws IOException {
diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRegexBasedRSGroupMembership.java b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRegexBasedRSGroupMembership.java
new file mode 100644
index 00000000000..3b694da6692
--- /dev/null
+++ b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRegexBasedRSGroupMembership.java
@@ -0,0 +1,1151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.rsgroup;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.IOException;
+import java.io.StringWriter;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.LocalHBaseCluster;
+import org.apache.hadoop.hbase.MiniHBaseCluster;
+import org.apache.hadoop.hbase.RSGroupTableAccessor;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.net.Address;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.DNS;
+import org.apache.hadoop.hbase.util.JVMClusterUtil;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
+
+/**
+ * Black-box tests for regex-based automatic RSGroup membership
+ * (hbase.rsgroup.regex.<groupname>=<regex>), driven entirely through the public
+ * {@link RSGroupAdmin} API, live RegionServer add/remove against the minicluster, and captured log
+ * output -- no reflection or visibility changes against {@link RSGroupInfoManagerImpl}.
+ */
+@Tag(MediumTests.TAG)
+public class TestRegexBasedRSGroupMembership extends TestRSGroupsBase {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestRegexBasedRSGroupMembership.class);
+ private static final String REGEX_PREFIX = "hbase.rsgroup.regex.";
+
+ private final List<JVMClusterUtil.RegionServerThread> fakeRegionServers = new ArrayList<>();
+ private final Set<String> regexConfigKeysSet = new HashSet<>();
+
+ @BeforeAll
+ public static void setUp() throws Exception {
+ setUpTestBeforeClass();
+ }
+
+ @AfterAll
+ public static void tearDown() throws Exception {
+ tearDownAfterClass();
+ }
+
+ @BeforeEach
+ public void beforeMethod(TestInfo testInfo) throws Exception {
+ setUpBeforeMethod(testInfo);
+ }
+
+ @AfterEach
+ public void afterMethod() throws Exception {
+ // Stop any fake-hostname RS this test started before handing back to the shared teardown,
+ // which only ever starts servers to top back up to NUM_SLAVES_BASE, never stops extras.
+ for (JVMClusterUtil.RegionServerThread rst : new ArrayList<>(fakeRegionServers)) {
+ stopFakeRegionServer(rst);
+ }
+ // The master Configuration object is shared across every test method in this class; undo
+ // any regex config this test added so it can't leak into the next method.
+ for (String key : regexConfigKeysSet) {
+ master.getConfiguration().unset(key);
+ TEST_UTIL.getConfiguration().unset(key);
+ }
+ regexConfigKeysSet.clear();
+ tearDownAfterMethod();
+ RSGroupBasedLoadBalancer balancer = (RSGroupBasedLoadBalancer) master.getLoadBalancer();
+ balancer.resetAssignmentCallFlagsForTest();
+ balancer.setFallbackEnabledForTest(false);
+ }
+
+ // ============================== helpers ==============================
+
+ /**
+ * Starts a genuinely new RegionServer process (thread) in the shared minicluster that reports
+ * itself under a caller-chosen hostname, so regex membership rules can target it individually.
+ * The base cluster's real RS all share one real, resolvable hostname (this machine's address);
+ * {@link RSRpcServices}'s constructor unconditionally resolves whatever hostname is configured
+ * (independent of any RPC bind-address override), so the caller-chosen hostname here must itself
+ * be a real, resolvable, bindable literal -- "127.0.0.1" and "localhost" both work and are
+ * guaranteed distinct from the base cluster's real hostname, which is exactly what every test in
+ * this class relies on to get a controllable, non-shared identity.
+ */
+ private JVMClusterUtil.RegionServerThread startFakeHostnameRS(String hostname) throws Exception {
+ Configuration rsConf = new Configuration(TEST_UTIL.getConfiguration());
+ rsConf.set(DNS.UNSAFE_RS_HOSTNAME_KEY, hostname);
+ rsConf.set("hbase.regionserver.ipc.address", "localhost");
+ // The shared TEST_UTIL configuration's regionserver port is never rewritten to an ephemeral
+ // value by minicluster startup (the base RS get their distinct ports through a separate,
+ // per-instance mechanism), so without this override every fake RS started here would try to
+ // bind the literal default port (16020) and collide with any other fake RS still alive.
+ rsConf.set(HConstants.REGIONSERVER_PORT, "0");
+ // Same issue for the embedded web UI (default port 16030); these fake RS don't need one.
+ rsConf.set(HConstants.REGIONSERVER_INFO_PORT, "-1");
+ LocalHBaseCluster localCluster = ((MiniHBaseCluster) cluster).hbaseCluster;
+ JVMClusterUtil.RegionServerThread rst =
+ localCluster.addRegionServer(rsConf, localCluster.getRegionServers().size());
+ rst.start();
+ rst.waitForServerOnline();
+ fakeRegionServers.add(rst);
+ final ServerName sn = rst.getRegionServer().getServerName();
+ TEST_UTIL.waitFor(WAIT_TIMEOUT,
+ () -> master.getServerManager().getOnlineServersList().contains(sn));
+ LOG.info("Started fake-hostname RS {}", sn);
+ return rst;
+ }
+
+ private void stopFakeRegionServer(JVMClusterUtil.RegionServerThread rst) throws Exception {
+ fakeRegionServers.remove(rst);
+ final ServerName sn = rst.getRegionServer().getServerName();
+ rst.getRegionServer().stop("test cleanup");
+ TEST_UTIL.waitFor(WAIT_TIMEOUT,
+ () -> !master.getServerManager().getOnlineServersList().contains(sn));
+ LOG.info("Stopped fake-hostname RS {}", sn);
+ }
+
+ private void killFakeRegionServer(JVMClusterUtil.RegionServerThread rst) throws Exception {
+ fakeRegionServers.remove(rst);
+ final ServerName sn = rst.getRegionServer().getServerName();
+ ((MiniHBaseCluster) cluster).killRegionServer(sn);
+ TEST_UTIL.waitFor(WAIT_TIMEOUT,
+ () -> !master.getServerManager().getOnlineServersList().contains(sn));
+ LOG.info("Killed fake-hostname RS {}", sn);
+ }
+
+ /**
+ * The listener thread that drives automatic regex-based reassignment only wakes up on an actual
+ * server add/remove event -- a bare config mutation is not enough. Forces a wake-up cycle
+ * (add+remove a bystander RS) so a config change just made takes effect without requiring an
+ * explicit RSGroupAdmin RPC. Only used by tests that *want* the automatic reconciliation to run
+ * and correct things; the trigger RS's own transient membership doesn't affect assertions
+ * elsewhere since those only check for specific other servers' addresses.
+ */
+ private void triggerListenerCycle() throws Exception {
+ JVMClusterUtil.RegionServerThread trigger = startFakeHostnameRS("127.0.0.1");
+ stopFakeRegionServer(trigger);
+ }
+
+ private void setRegex(String groupName, String regex) {
+ String key = REGEX_PREFIX + groupName;
+ master.getConfiguration().set(key, regex);
+ // The master's Configuration is a distinct object from TEST_UTIL's (the minicluster copies
+ // configuration per daemon), so mirror the change there too --
+ // VerifyingRSGroupAdminClient#verify
+ // reads it to know which servers' membership is regex-driven.
+ TEST_UTIL.getConfiguration().set(key, regex);
+ regexConfigKeysSet.add(key);
+ }
+
+ /**
+ * Tears down a regex-governed group: the regex is first dropped on the master only -- TEST_UTIL's
+ * copy still tells VerifyingRSGroupAdminClient which servers' membership is regex-driven -- and
+ * then the group is emptied and removed.
+ */
+ private void removeRegexGroup(String groupName) throws Exception {
+ // Move the tables out while the regex is still active: the flush this triggers must keep not
+ // persisting the group's servers.
+ rsGroupAdmin.moveTables(rsGroupAdmin.getRSGroupInfo(groupName).getTables(),
+ RSGroupInfo.DEFAULT_GROUP);
+ String key = REGEX_PREFIX + groupName;
+ master.getConfiguration().unset(key);
+ try {
+ removeGroup(groupName);
+ } finally {
+ clearRegex(groupName);
+ }
+ }
+
+ /** Reads a group's servers the way RegionMover and the master web UI do. */
+ private Set<Address> rsGroupTableAccessorServers(String groupName) throws IOException {
+ RSGroupInfo info =
+ RSGroupTableAccessor.getRSGroupInfo(TEST_UTIL.getConnection(), Bytes.toBytes(groupName));
+ return info == null ? new HashSet<>() : new HashSet<>(info.getServers());
+ }
+
+ /**
+ * Fails over to a freshly started master (the minicluster's masters are threads): a backup is
+ * started, the active master is stopped, and the shared handles are re-pointed.
+ */
+ private void restartMaster() throws Exception {
+ MiniHBaseCluster miniCluster = TEST_UTIL.getMiniHBaseCluster();
+ int oldIndex = -1;
+ List<JVMClusterUtil.MasterThread> masters = miniCluster.getMasterThreads();
+ for (int i = 0; i < masters.size(); i++) {
+ if (masters.get(i).getMaster() == master) {
+ oldIndex = i;
+ }
+ }
+ miniCluster.startMaster();
+ miniCluster.stopMaster(oldIndex);
+ miniCluster.waitOnMaster(oldIndex);
+ assertTrue(miniCluster.waitForActiveAndReadyMaster(WAIT_TIMEOUT));
+ // Clients bootstrap from the configured master address, which is now the dead master's.
+ TEST_UTIL.getConfiguration().set(HConstants.MASTER_ADDRS_KEY,
+ miniCluster.getMaster().getServerName().getAddress().toString());
+ TEST_UTIL.closeConnection();
+ initialize();
+ }
+
+ private void clearRegex(String groupName) {
+ String key = REGEX_PREFIX + groupName;
+ master.getConfiguration().unset(key);
+ TEST_UTIL.getConfiguration().unset(key);
+ regexConfigKeysSet.remove(key);
+ }
+
+ private Address addressOf(JVMClusterUtil.RegionServerThread rst) {
+ return rst.getRegionServer().getServerName().getAddress();
+ }
+
+ private static class LogCapturer {
+ private final StringWriter sw = new StringWriter();
+ private final org.apache.logging.log4j.core.appender.WriterAppender appender;
+ private final org.apache.logging.log4j.core.Logger logger;
+
+ LogCapturer(org.apache.logging.log4j.core.Logger logger) {
+ this.logger = logger;
+ this.appender = org.apache.logging.log4j.core.appender.WriterAppender.newBuilder()
+ .setName("test-regex-membership").setTarget(sw).build();
+ this.appender.start();
+ this.logger.addAppender(this.appender);
+ }
+
+ String getOutput() {
+ return sw.toString();
+ }
+
+ void stopCapturing() {
+ this.logger.removeAppender(this.appender);
+ this.appender.stop();
+ }
+ }
+
+ private static LogCapturer captureRSGroupInfoManagerLog() {
+ return new LogCapturer(
+ (org.apache.logging.log4j.core.Logger) org.apache.logging.log4j.LogManager
+ .getLogger(RSGroupInfoManagerImpl.class));
+ }
+
+ // ============================== tests ==============================
+
+ @Test
+ public void testNoRegexConfigBaseline() throws Exception {
+ // No hbase.rsgroup.regex.* configured at all: every server must remain in default, exactly
+ // as it would without this feature. This is also implicitly exercised by every other test in
+ // this module, but pin it down explicitly for this feature.
+ RSGroupInfo defaultGroup = rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP);
+ assertEquals(NUM_SLAVES_BASE, defaultGroup.getServers().size());
+ }
+
+ @Test
+ public void testSingleRegexAutoJoinOnLiveAdd() throws Exception {
+ String groupName = getGroupName("auto");
+ rsGroupAdmin.addRSGroup(groupName);
+ setRegex(groupName, "127\\.0\\.0\\.1");
+
+ JVMClusterUtil.RegionServerThread rst = startFakeHostnameRS("127.0.0.1");
+ Address addr = addressOf(rst);
+ try {
+ TEST_UTIL.waitFor(WAIT_TIMEOUT,
+ () -> rsGroupAdmin.getRSGroupInfo(groupName).getServers().contains(addr));
+ // No explicit moveServers call was made -- the server auto-joined purely from the
+ // ServerEventsListenerThread reacting to its own arrival.
+ assertFalse(
+ rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().contains(addr));
+ } finally {
+ removeRegexGroup(groupName);
+ }
+ }
+
+ @Test
+ public void testRegexWithQuantifiersMatches() throws Exception {
+ String groupName = getGroupName("quant");
+ rsGroupAdmin.addRSGroup(groupName);
+ setRegex(groupName, "127\\.0\\.0\\.\\d{1,3}");
+
+ JVMClusterUtil.RegionServerThread rst = startFakeHostnameRS("127.0.0.1");
+ Address addr = addressOf(rst);
+ try {
+ TEST_UTIL.waitFor(WAIT_TIMEOUT,
+ () -> rsGroupAdmin.getRSGroupInfo(groupName).getServers().contains(addr));
+ // No explicit moveServers call was made -- the server auto-joined purely from the
+ // ServerEventsListenerThread reacting to its own arrival.
+ assertFalse(
+ rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().contains(addr));
+ } finally {
+ removeRegexGroup(groupName);
+ }
+ }
+
+ @Test
+ public void testFullStringMatchSemantics() throws Exception {
+ // The regex is a strict prefix of the hostname; full-string Pattern#matches semantics must
+ // NOT treat this as a match.
+ String groupName = getGroupName("prefix");
+ rsGroupAdmin.addRSGroup(groupName);
+ // Strict prefix of "127.0.0.1" -- must NOT match under full-string Pattern#matches semantics.
+ setRegex(groupName, "127\\.0\\.0");
+
+ JVMClusterUtil.RegionServerThread rst = startFakeHostnameRS("127.0.0.1");
+ Address addr = addressOf(rst);
+ try {
+ TEST_UTIL.waitFor(WAIT_TIMEOUT,
+ () -> rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().contains(addr));
+ assertFalse(rsGroupAdmin.getRSGroupInfo(groupName).getServers().contains(addr));
+ } finally {
+ removeRegexGroup(groupName);
+ }
+ }
+
+ @Test
+ public void testRegexTargetingDefaultGroupIsNoOpAndWarns() throws Exception {
+ LogCapturer capturer = captureRSGroupInfoManagerLog();
+ try {
+ setRegex(RSGroupInfo.DEFAULT_GROUP, "127\\.0\\.0\\..*");
+ JVMClusterUtil.RegionServerThread rst = startFakeHostnameRS("127.0.0.1");
+ Address addr = addressOf(rst);
+ TEST_UTIL.waitFor(WAIT_TIMEOUT,
+ () -> rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().contains(addr));
+ TEST_UTIL.waitFor(WAIT_TIMEOUT, () -> capturer.getOutput()
+ .contains("cannot target the reserved '" + RSGroupInfo.DEFAULT_GROUP + "' group"));
+ } finally {
+ capturer.stopCapturing();
+ clearRegex(RSGroupInfo.DEFAULT_GROUP);
+ }
+ }
+
+ @Test
+ public void testInvalidRegexSyntaxIgnoredAndWarns() throws Exception {
+ String groupName = getGroupName("badregex");
+ rsGroupAdmin.addRSGroup(groupName);
+ LogCapturer capturer = captureRSGroupInfoManagerLog();
+ try {
+ setRegex(groupName, "[");
+ JVMClusterUtil.RegionServerThread rst = startFakeHostnameRS("127.0.0.1");
+ Address addr = addressOf(rst);
+ TEST_UTIL.waitFor(WAIT_TIMEOUT,
+ () -> rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().contains(addr));
+ TEST_UTIL.waitFor(WAIT_TIMEOUT,
+ () -> capturer.getOutput().contains("Invalid regex '[' for RSGroup '" + groupName + "'"));
+ } finally {
+ capturer.stopCapturing();
+ removeRegexGroup(groupName);
+ }
+ }
+
+ @Test
+ public void testInvalidGroupNameKeyIgnoredAndWarns() throws Exception {
+ LogCapturer capturer = captureRSGroupInfoManagerLog();
+ try {
+ setRegex("not a valid name", "127\\.0\\.0\\..*");
+ JVMClusterUtil.RegionServerThread rst = startFakeHostnameRS("127.0.0.1");
+ Address addr = addressOf(rst);
+ TEST_UTIL.waitFor(WAIT_TIMEOUT,
+ () -> rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().contains(addr));
+ TEST_UTIL.waitFor(WAIT_TIMEOUT,
+ () -> capturer.getOutput().contains("is not a valid RSGroup name"));
+ } finally {
+ capturer.stopCapturing();
+ clearRegex("not a valid name");
+ }
+ }
+
+ @Test
+ public void testListenerDoesNotStickToStaleServer() throws Exception {
+ // A server leaves, then a *different* server with a matching hostname joins later; the
+ // listener thread must not get stuck comparing against a stale previous-assignments snapshot.
+ String groupName = getGroupName("stale");
+ rsGroupAdmin.addRSGroup(groupName);
+ setRegex(groupName, "127\\.0\\.0\\..*");
+ try {
+ JVMClusterUtil.RegionServerThread first = startFakeHostnameRS("127.0.0.1");
+ Address firstAddr = addressOf(first);
+ TEST_UTIL.waitFor(WAIT_TIMEOUT,
+ () -> rsGroupAdmin.getRSGroupInfo(groupName).getServers().contains(firstAddr));
+
+ stopFakeRegionServer(first);
+
+ // Different startcode (fresh ServerName) under the same matching hostname.
+ JVMClusterUtil.RegionServerThread second = startFakeHostnameRS("127.0.0.1");
+ Address secondAddr = addressOf(second);
+ TEST_UTIL.waitFor(WAIT_TIMEOUT,
+ () -> rsGroupAdmin.getRSGroupInfo(groupName).getServers().contains(secondAddr));
+ } finally {
+ removeRegexGroup(groupName);
+ }
+ }
+
+ @Test
+ public void testDanglingGroupFallsBackThenResolvesOnceGroupCreated() throws Exception {
+ String groupName = getGroupName("dangling");
+ LogCapturer capturer = captureRSGroupInfoManagerLog();
+ try {
+ // Group does not exist yet when the regex is configured and the matching RS joins.
+ setRegex(groupName, "127\\.0\\.0\\..*");
+ JVMClusterUtil.RegionServerThread rst = startFakeHostnameRS("127.0.0.1");
+ Address addr = addressOf(rst);
+ TEST_UTIL.waitFor(WAIT_TIMEOUT,
+ () -> rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().contains(addr));
+ TEST_UTIL.waitFor(WAIT_TIMEOUT, () -> capturer.getOutput()
+ .contains("RSGroup '" + groupName + "' does not exist -- create it with addRSGroup"));
+
+ // Now create the group; on the next cycle the server should migrate in, no restart needed.
+ rsGroupAdmin.addRSGroup(groupName);
+ triggerListenerCycle();
+ TEST_UTIL.waitFor(WAIT_TIMEOUT,
+ () -> rsGroupAdmin.getRSGroupInfo(groupName).getServers().contains(addr));
+ } finally {
+ capturer.stopCapturing();
+ removeRegexGroup(groupName);
+ }
+ }
+
+ @Test
+ public void testAmbiguousMatchFallsBackThenResolvesOnceOverlapFixed() throws Exception {
+ String groupOne = getGroupName("ambig1");
+ String groupTwo = getGroupName("ambig2");
+ rsGroupAdmin.addRSGroup(groupOne);
+ rsGroupAdmin.addRSGroup(groupTwo);
+ LogCapturer capturer = captureRSGroupInfoManagerLog();
+ try {
+ setRegex(groupOne, "127\\.0\\.0\\..*");
+ setRegex(groupTwo, "127\\.0\\.0\\.1");
+
+ JVMClusterUtil.RegionServerThread rst = startFakeHostnameRS("127.0.0.1");
+ Address addr = addressOf(rst);
+ TEST_UTIL.waitFor(WAIT_TIMEOUT,
+ () -> rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().contains(addr));
+ TEST_UTIL.waitFor(WAIT_TIMEOUT,
+ () -> capturer.getOutput().contains("matches regexes for multiple RSGroups"));
+ assertFalse(rsGroupAdmin.getRSGroupInfo(groupOne).getServers().contains(addr));
+ assertFalse(rsGroupAdmin.getRSGroupInfo(groupTwo).getServers().contains(addr));
+
+ // Fix the overlap
+ setRegex(groupOne, "10\\.0\\.0\\..*");
+ triggerListenerCycle();
+ TEST_UTIL.waitFor(WAIT_TIMEOUT,
+ () -> rsGroupAdmin.getRSGroupInfo(groupTwo).getServers().contains(addr));
+ } finally {
+ capturer.stopCapturing();
+ removeRegexGroup(groupOne);
+ removeRegexGroup(groupTwo);
+ }
+ }
+
+ @Test
+ public void testRegexCoveringEveryServerEmptiesDefaultAndWarns() throws Exception {
+ // Regex placement is always applied: a catch-all regex matches every online server (all base
+ // RS share one real hostname), so they all move into the regex group and default is left
+ // empty. That is a warning about an unusual configuration, not an error.
+ // (Explicit moveServers can never empty default -- an unrelated, longstanding
+ // RSGroupInfoManagerImpl#moveServers guard keeps >=1 server there -- so the empty-default
+ // state is driven purely through the automatic regex-reconciliation path.)
+ String regexGroup = getGroupName("emptydefault");
+ rsGroupAdmin.addRSGroup(regexGroup);
+ LogCapturer capturer = captureRSGroupInfoManagerLog();
+ try {
+ setRegex(regexGroup, ".*");
+ triggerListenerCycle();
+
+ TEST_UTIL.waitFor(WAIT_TIMEOUT,
+ () -> capturer.getOutput().contains("leaves RSGroup 'default' with no online servers"));
+ assertTrue(capturer.getOutput().contains(RSGroupBasedLoadBalancer.FALLBACK_GROUP_ENABLE_KEY));
+ TEST_UTIL.waitFor(WAIT_TIMEOUT,
+ () -> rsGroupAdmin.getRSGroupInfo(regexGroup).getServers().size() == NUM_SLAVES_BASE);
+ assertTrue(rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().isEmpty());
+
+ // Narrow the regex so it only targets a new server, not the base cluster's shared
+ // hostname -- the base servers are unmatched again and return to default.
+ setRegex(regexGroup, "127\\.0\\.0\\..*");
+ JVMClusterUtil.RegionServerThread rst = startFakeHostnameRS("127.0.0.1");
+ Address addr = addressOf(rst);
+ TEST_UTIL.waitFor(WAIT_TIMEOUT,
+ () -> rsGroupAdmin.getRSGroupInfo(regexGroup).getServers().contains(addr)
+ && rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().size()
+ == NUM_SLAVES_BASE);
+ } finally {
+ capturer.stopCapturing();
+ removeRegexGroup(regexGroup);
+ }
+ }
+
+ @Test
+ public void testCatchAllRegexMovesEveryServerAndGroupTablesStayAssigned() throws Exception {
+ // A regex group with a live member and a bound table is broadened to a catch-all. Every
+ // server is placed in its regex group (default is left empty), and the group's table stays
+ // fully assigned on its original member.
+ String groupName = getGroupName("catchall");
+ rsGroupAdmin.addRSGroup(groupName);
+ setRegex(groupName, "127\\.0\\.0\\.1");
+ RSGroupBasedLoadBalancer balancer = (RSGroupBasedLoadBalancer) master.getLoadBalancer();
+ balancer.setFallbackEnabledForTest(true);
+ try {
+ JVMClusterUtil.RegionServerThread rst = startFakeHostnameRS("127.0.0.1");
+ Address addr = addressOf(rst);
+ TEST_UTIL.waitFor(WAIT_TIMEOUT,
+ () -> rsGroupAdmin.getRSGroupInfo(groupName).getServers().contains(addr));
+ ServerName sn = getServerName(addr);
+
+ TEST_UTIL.createMultiRegionTable(tableName, Bytes.toBytes("f"), 5);
+ TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
+ rsGroupAdmin.moveTables(Sets.newHashSet(tableName), groupName);
+ assertEquals(5, getTableServerRegionMap().get(tableName).get(sn).size());
+
+ setRegex(groupName, ".*");
+ triggerListenerCycle();
+ TEST_UTIL.waitFor(WAIT_TIMEOUT, () -> {
+ Set<Address> servers = rsGroupAdmin.getRSGroupInfo(groupName).getServers();
+ return servers.contains(addr) && servers.size() == NUM_SLAVES_BASE + 1
+ && rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().isEmpty();
+ });
+ assertEquals(5, getTableServerRegionMap().get(tableName).get(sn).size());
+ } finally {
+ balancer.setFallbackEnabledForTest(false);
+ // moveTables requires the target group to have a server, and default was emptied above, so
+ // narrow the regex back first to let the base servers return to default.
+ setRegex(groupName, "127\\.0\\.0\\.1");
+ triggerListenerCycle();
+ TEST_UTIL.waitFor(WAIT_TIMEOUT,
+ () -> rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().size()
+ == NUM_SLAVES_BASE);
+ removeRegexGroup(groupName);
+ }
+ }
+
+ @Test
+ public void testStoredServersOfRegexGovernedGroupAreIgnoredOnRefresh() throws Exception {
+ String groupName = getGroupName("stale");
+ rsGroupAdmin.addRSGroup(groupName);
+ setRegex(groupName, "127\\.0\\.0\\.1");
+ try {
+ JVMClusterUtil.RegionServerThread ipRs = startFakeHostnameRS("127.0.0.1");
+ JVMClusterUtil.RegionServerThread nameRs = startFakeHostnameRS("localhost");
+ Address ipAddr = addressOf(ipRs);
+ Address nameAddr = addressOf(nameRs);
+ TEST_UTIL.waitFor(WAIT_TIMEOUT, () -> rsGroupTableAccessorServers(groupName).contains(ipAddr)
+ && rsGroupTableAccessorServers(RSGroupInfo.DEFAULT_GROUP).contains(nameAddr));
+ assertFalse(rsGroupTableAccessorServers(groupName).contains(nameAddr));
+
+ // Point the regex at the other server and fail over. A new master copies the minicluster
+ // configuration, so set it there too. The table still stores the old membership, which
+ // refresh must ignore.
+ setRegex(groupName, "localhost");
+ TEST_UTIL.getMiniHBaseCluster().getConfiguration().set(REGEX_PREFIX + groupName, "localhost");
+ restartMaster();
+
+ TEST_UTIL.waitFor(WAIT_TIMEOUT,
+ () -> rsGroupAdmin.getRSGroupInfo(groupName).getServers().contains(nameAddr));
+ assertFalse(rsGroupAdmin.getRSGroupInfo(groupName).getServers().contains(ipAddr));
+ // The startup flush rewrites the stored copy from the recomputed membership.
+ TEST_UTIL.waitFor(WAIT_TIMEOUT,
+ () -> rsGroupTableAccessorServers(groupName).contains(nameAddr)
+ && !rsGroupTableAccessorServers(groupName).contains(ipAddr)
+ && rsGroupTableAccessorServers(RSGroupInfo.DEFAULT_GROUP).contains(ipAddr));
+ } finally {
+ TEST_UTIL.getMiniHBaseCluster().getConfiguration().unset(REGEX_PREFIX + groupName);
+ removeRegexGroup(groupName);
+ }
+ }
+
+ @Test
+ public void testRegexGovernedGroupMembershipIsPersistedForTableReaders() throws Exception {
+ // Regex-governed membership is computed on the fly, so it is visible even while hbase:rsgroup
+ // is unavailable. Once the table is back, a flush stores it so that direct readers of the
+ // table (RSGroupTableAccessor) see it too.
+ String groupName = getGroupName("persist");
+ rsGroupAdmin.addRSGroup(groupName);
+ setRegex(groupName, "127\\.0\\.0\\.1");
+ boolean tableDisabled = false;
+ try {
+ TEST_UTIL.getAdmin().disableTable(RSGroupInfoManagerImpl.RSGROUP_TABLE_NAME);
+ tableDisabled = true;
+
+ JVMClusterUtil.RegionServerThread rst = startFakeHostnameRS("127.0.0.1");
+ Address addr = addressOf(rst);
+ TEST_UTIL.waitFor(WAIT_TIMEOUT,
+ () -> rsGroupAdmin.getRSGroupInfo(groupName).getServers().contains(addr));
+
+ TEST_UTIL.getAdmin().enableTable(RSGroupInfoManagerImpl.RSGROUP_TABLE_NAME);
+ tableDisabled = false;
+ // Any group mutation flushes every group.
+ String flushGroup = getGroupName("persistflush");
+ rsGroupAdmin.addRSGroup(flushGroup);
+ removeGroup(flushGroup);
+
+ assertTrue(rsGroupAdmin.getRSGroupInfo(groupName).getServers().contains(addr));
+ assertTrue(rsGroupTableAccessorServers(groupName).contains(addr));
+ } finally {
+ if (tableDisabled) {
+ TEST_UTIL.getAdmin().enableTable(RSGroupInfoManagerImpl.RSGROUP_TABLE_NAME);
+ }
+ removeRegexGroup(groupName);
+ }
+ }
+
+ @Test
+ public void testMoveContradictingRegexIsRejected() throws Exception {
+ String groupName = getGroupName("reject");
+ String otherGroup = getGroupName("rejectother");
+ rsGroupAdmin.addRSGroup(groupName);
+ rsGroupAdmin.addRSGroup(otherGroup);
+ setRegex(groupName, "127\\.0\\.0\\..*");
+ try {
+ // A server whose hostname does not match the regex cannot be moved into the regex group ...
+ JVMClusterUtil.RegionServerThread nonMatching = startFakeHostnameRS("localhost");
+ Address nonMatchingAddr = addressOf(nonMatching);
+ TEST_UTIL.waitFor(WAIT_TIMEOUT, () -> rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP)
+ .getServers().contains(nonMatchingAddr));
+ assertThrows(IOException.class,
+ () -> rsGroupAdmin.moveServers(Sets.newHashSet(nonMatchingAddr), groupName));
+ assertTrue(rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers()
+ .contains(nonMatchingAddr));
+ assertFalse(rsGroupAdmin.getRSGroupInfo(groupName).getServers().contains(nonMatchingAddr));
+
+ // ... but can still be placed in any other group.
+ rsGroupAdmin.moveServers(Sets.newHashSet(nonMatchingAddr), otherGroup);
+ assertTrue(rsGroupAdmin.getRSGroupInfo(otherGroup).getServers().contains(nonMatchingAddr));
+ rsGroupAdmin.moveServers(Sets.newHashSet(nonMatchingAddr), RSGroupInfo.DEFAULT_GROUP);
+
+ // A server the regex claims cannot be moved out of, or to any group other than, its group.
+ JVMClusterUtil.RegionServerThread matching = startFakeHostnameRS("127.0.0.1");
+ Address matchingAddr = addressOf(matching);
+ TEST_UTIL.waitFor(WAIT_TIMEOUT,
+ () -> rsGroupAdmin.getRSGroupInfo(groupName).getServers().contains(matchingAddr));
+ assertThrows(IOException.class,
+ () -> rsGroupAdmin.moveServers(Sets.newHashSet(matchingAddr), RSGroupInfo.DEFAULT_GROUP));
+ assertThrows(IOException.class,
+ () -> rsGroupAdmin.moveServers(Sets.newHashSet(matchingAddr), otherGroup));
+ assertTrue(rsGroupAdmin.getRSGroupInfo(groupName).getServers().contains(matchingAddr));
+ } finally {
+ removeGroup(otherGroup);
+ removeRegexGroup(groupName);
+ }
+ }
+
+ @Test
+ public void testUnrelatedAdminOpsSucceedUnderActiveRegex() throws Exception {
+ String regexGroup = getGroupName("activeregex");
+ String otherGroup = getGroupName("other");
+ rsGroupAdmin.addRSGroup(regexGroup);
+ setRegex(regexGroup, "127\\.0\\.0\\..*");
+ try {
+ JVMClusterUtil.RegionServerThread rst = startFakeHostnameRS("127.0.0.1");
+ Address addr = addressOf(rst);
+ TEST_UTIL.waitFor(WAIT_TIMEOUT,
+ () -> rsGroupAdmin.getRSGroupInfo(regexGroup).getServers().contains(addr));
+
+ // Unrelated admin operations must not be spuriously blocked by the invariant check.
+ // moveTables requires the target group to already have at least one server, so use a
+ // base (non-regex-matched, admin-manageable) server rather than an empty new group.
+ RSGroupInfo otherGroupInfo = addGroup(otherGroup, 1);
+ Set<Address> otherGroupServers = otherGroupInfo.getServers();
+ TEST_UTIL.createTable(tableName, Bytes.toBytes("f"));
+ rsGroupAdmin.moveTables(Sets.newHashSet(tableName), otherGroup);
+ assertTrue(rsGroupAdmin.getRSGroupInfo(otherGroup).getTables().contains(tableName));
+
+ String renamedGroup = otherGroup + "_renamed";
+ rsGroupAdmin.renameRSGroup(otherGroup, renamedGroup);
+ RSGroupInfo renamedGroupInfo = rsGroupAdmin.getRSGroupInfo(renamedGroup);
+ assertEquals(otherGroupServers, renamedGroupInfo.getServers());
+ assertTrue(renamedGroupInfo.getTables().contains(tableName));
+ assertFalse(
+ rsGroupAdmin.listRSGroups().stream().anyMatch(g -> g.getName().equals(otherGroup)));
+
+ rsGroupAdmin.moveTables(rsGroupAdmin.getRSGroupInfo(renamedGroup).getTables(),
+ RSGroupInfo.DEFAULT_GROUP);
+ assertTrue(
+ rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getTables().contains(tableName));
+ assertTrue(rsGroupAdmin.getRSGroupInfo(renamedGroup).getTables().isEmpty());
+
+ rsGroupAdmin.moveServers(rsGroupAdmin.getRSGroupInfo(renamedGroup).getServers(),
+ RSGroupInfo.DEFAULT_GROUP);
+ Set<Address> defaultServersAfterMoveBack =
+ rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers();
+ for (Address server : otherGroupServers) {
+ assertTrue(defaultServersAfterMoveBack.contains(server));
+ }
+ assertTrue(rsGroupAdmin.getRSGroupInfo(renamedGroup).getServers().isEmpty());
+
+ rsGroupAdmin.removeRSGroup(renamedGroup);
+ assertFalse(
+ rsGroupAdmin.listRSGroups().stream().anyMatch(g -> g.getName().equals(renamedGroup)));
+
+ // Confirm the unrelated operations above never disturbed the active regex-governed group.
+ assertTrue(rsGroupAdmin.getRSGroupInfo(regexGroup).getServers().contains(addr));
+ } finally {
+ TEST_UTIL.deleteTable(tableName);
+ removeRegexGroup(regexGroup);
+ }
+ }
+
+ @Test
+ public void testAutomaticMoveDoesNotFireCoprocessorHooks() throws Exception {
+ String groupName = getGroupName("hooks");
+ rsGroupAdmin.addRSGroup(groupName);
+ setRegex(groupName, "127\\.0\\.0\\..*");
+ try {
+ assertFalse(observer.preMoveServersCalled);
+ JVMClusterUtil.RegionServerThread rst = startFakeHostnameRS("127.0.0.1");
+ Address addr = addressOf(rst);
+ TEST_UTIL.waitFor(WAIT_TIMEOUT,
+ () -> rsGroupAdmin.getRSGroupInfo(groupName).getServers().contains(addr));
+ // Purely automatic, listener-thread-driven reassignment must not go through the
+ // RSGroupAdminServer RPC surface, so the moveServers coprocessor hooks must not fire.
+ assertFalse(observer.preMoveServersCalled);
+ assertFalse(observer.postMoveServersCalled);
+
+ // An explicit admin-driven move, by contrast, does fire the hooks. Use one of the
+ // base cluster's own (non-regex-matched) RS for this, rather than disturbing the
+ // regex-matched fake RS started above, which keeps running under its active regex.
+ String otherGroup = getGroupName("hooksother");
+ rsGroupAdmin.addRSGroup(otherGroup);
+ Address baseServerAddr =
+ rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().iterator().next();
+ rsGroupAdmin.moveServers(Sets.newHashSet(baseServerAddr), otherGroup);
+ assertTrue(observer.preMoveServersCalled);
+ assertTrue(observer.postMoveServersCalled);
+ removeGroup(otherGroup);
+
+ // The unrelated explicit move above must not have disturbed the regex-governed group.
+ assertTrue(rsGroupAdmin.getRSGroupInfo(groupName).getServers().contains(addr));
+ } finally {
+ removeRegexGroup(groupName);
+ }
+ }
+
+ @Test
+ public void testRefreshRemovesRegexMatchedServerFromStoredAdminGroup() throws Exception {
+ // A server stored in an admin-managed group before a regex claimed it must not end up in two
+ // groups after a manager restart. refresh cannot reject (it would wedge startup), so the regex
+ // wins in memory and a WARN is logged.
+ String adminGroup = getGroupName("refreshadmin");
+ String regexGroup = getGroupName("refreshregex");
+ rsGroupAdmin.addRSGroup(adminGroup);
+ rsGroupAdmin.addRSGroup(regexGroup);
+ LogCapturer capturer = captureRSGroupInfoManagerLog();
+ try {
+ JVMClusterUtil.RegionServerThread rst = startFakeHostnameRS("127.0.0.1");
+ Address addr = addressOf(rst);
+ TEST_UTIL.waitFor(WAIT_TIMEOUT,
+ () -> rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().contains(addr));
+ rsGroupAdmin.moveServers(Sets.newHashSet(addr), adminGroup);
+
+ // The regex arrives while the server is already stored in the admin group. A new master
+ // copies the minicluster configuration, so set it there too before the failover.
+ setRegex(regexGroup, "127\\.0\\.0\\..*");
+ TEST_UTIL.getMiniHBaseCluster().getConfiguration().set(REGEX_PREFIX + regexGroup,
+ "127\\.0\\.0\\..*");
+ restartMaster();
+ TEST_UTIL.waitFor(WAIT_TIMEOUT,
+ () -> rsGroupAdmin.getRSGroupInfo(regexGroup).getServers().contains(addr));
+ assertFalse(rsGroupAdmin.getRSGroupInfo(adminGroup).getServers().contains(addr));
+ assertFalse(
+ rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().contains(addr));
+ assertTrue(capturer.getOutput().contains("are stored in RSGroup '" + adminGroup + "'"));
+
+ // The startup flush replaces the stale stored copy: the admin group no longer holds the
+ // server and the regex-governed group stores it.
+ TEST_UTIL.waitFor(WAIT_TIMEOUT, () -> !rsGroupTableAccessorServers(adminGroup).contains(addr)
+ && rsGroupTableAccessorServers(regexGroup).contains(addr));
+ } finally {
+ capturer.stopCapturing();
+ TEST_UTIL.getMiniHBaseCluster().getConfiguration().unset(REGEX_PREFIX + regexGroup);
+ removeRegexGroup(regexGroup);
+ removeGroup(adminGroup);
+ }
+ }
+
+ @Test
+ public void testRegexConfigDriftDoesNotBlockUnrelatedAdminOps() throws Exception {
+ // A regex that contradicts where a server was explicitly placed must not block any unrelated
+ // admin operation.
+ String groupName = getGroupName("drift");
+ String bystanderGroup = getGroupName("bystander");
+ rsGroupAdmin.addRSGroup(groupName);
+ rsGroupAdmin.addRSGroup(bystanderGroup);
+ String renamedBystander = bystanderGroup + "_renamed";
+ try {
+ JVMClusterUtil.RegionServerThread rst = startFakeHostnameRS("127.0.0.1");
+ Address addr = addressOf(rst);
+ TEST_UTIL.waitFor(WAIT_TIMEOUT,
+ () -> rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().contains(addr));
+
+ // Legal at the time: no regex governs groupName yet.
+ rsGroupAdmin.moveServers(Sets.newHashSet(addr), groupName);
+
+ // Now introduce a regex for groupName that does NOT match the server already placed there.
+ // No listener cycle runs, so the contradiction stays un-reconciled.
+ setRegex(groupName, "10\\.0\\.0\\..*");
+
+ rsGroupAdmin.renameRSGroup(bystanderGroup, renamedBystander);
+ assertTrue(
+ rsGroupAdmin.listRSGroups().stream().anyMatch(g -> g.getName().equals(renamedBystander)));
+ } finally {
+ removeRegexGroup(groupName);
+ removeGroup(
+ rsGroupAdmin.getRSGroupInfo(bystanderGroup) != null ? bystanderGroup : renamedBystander);
+ }
+ }
+
+ @Test
+ public void testNewlyJoinedRegexMemberReceivesRegionsOnBalance() throws Exception {
+ // A new RS auto-joins an already-regex-governed group while the group's sole existing
+ // member holds every region of a table bound to that group; running the balancer must then
+ // hand some of those regions to the newly-joined RS too. The first-member baseline is
+ // established by moveTables itself: RSGroupAdminServer#moveTables synchronously relocates
+ // every region of a table with no RSGroup-scoped descriptor yet onto the target group's
+ // servers (see moveTableRegionsToGroup, which blocks on the move futures before returning),
+ // so with sn1 as the sole group member at that point all regions are already there once
+ // moveTables returns -- no separate balance step is needed for that part. Only the
+ // second-member handoff genuinely needs an explicit balance: adding a group member does not
+ // by itself retrigger any region movement.
+ String groupName = getGroupName("balancenewmember");
+ rsGroupAdmin.addRSGroup(groupName);
+ // "127.0.0.1" and "localhost" are the only two distinct, real, independently-reachable
+ // identities startFakeHostnameRS supports (its RPC bind address is always "localhost"
+ // regardless of advertised hostname).
+ setRegex(groupName, "127\\.0\\.0\\.1|localhost");
+ try {
+ JVMClusterUtil.RegionServerThread rst1 = startFakeHostnameRS("127.0.0.1");
+ Address addr1 = addressOf(rst1);
+ TEST_UTIL.waitFor(WAIT_TIMEOUT,
+ () -> rsGroupAdmin.getRSGroupInfo(groupName).getServers().contains(addr1));
+ ServerName sn1 = getServerName(addr1);
+
+ TEST_UTIL.createMultiRegionTable(tableName, Bytes.toBytes("f"), 10);
+ TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
+ rsGroupAdmin.moveTables(Sets.newHashSet(tableName), groupName);
+ Map<ServerName, List<String>> perServerAfterMove = getTableServerRegionMap().get(tableName);
+ assertEquals(10, perServerAfterMove.get(sn1).size());
+
+ admin.balancerSwitch(true, true);
+ try {
+ // A second RS matching the same regex joins while the group is already active; it must
+ // auto-join with no explicit RPC.
+ JVMClusterUtil.RegionServerThread rst2 = startFakeHostnameRS("localhost");
+ Address addr2 = addressOf(rst2);
+ TEST_UTIL.waitFor(WAIT_TIMEOUT,
+ () -> rsGroupAdmin.getRSGroupInfo(groupName).getServers().contains(addr2));
+ ServerName sn2 = getServerName(addr2);
+
+ rsGroupAdmin.balanceRSGroup(groupName);
+ TEST_UTIL.waitFor(WAIT_TIMEOUT, () -> {
+ Map<ServerName, List<String>> perServer = getTableServerRegionMap().get(tableName);
+ if (perServer == null) {
+ return false;
+ }
+ List<String> onNewMember = perServer.get(sn2);
+ if (onNewMember == null || onNewMember.isEmpty()) {
+ return false;
+ }
+ int total = perServer.values().stream().mapToInt(List::size).sum();
+ return total == 10;
+ });
+ } finally {
+ admin.balancerSwitch(false, true);
+ }
+ } finally {
+ removeRegexGroup(groupName);
+ }
+ }
+
+ @Test
+ public void testRestartTriggeredMembershipConfinesTableToGroupServers() throws Exception {
+ // RS join normally (no regex active yet), landing in default like any other RS. Only
+ // afterward is a regex configured targeting their hostnames; a bare config change does not
+ // reconcile membership by itself, so an actual lifecycle event -- restarting one of the two
+ // RS -- is used to force the listener thread to notice and migrate both into the new group
+ // (the recompute triggered by one server's add/remove event considers every online server,
+ // not just the one that triggered it). A table is then bound to that group, and its regions
+ // must land only on the group's members, with those members hosting only that table's
+ // regions -- a bidirectional isolation check.
+ String groupName = getGroupName("restartisolation");
+ // "127.0.0.1" and "localhost" are the only two distinct, real, independently-reachable
+ // identities startFakeHostnameRS supports (its RPC bind address is always "localhost"
+ // regardless of advertised hostname -- an arbitrary loopback alias like "127.0.0.2" would
+ // advertise an address nothing is actually listening on, stalling any real region-open RPC).
+ JVMClusterUtil.RegionServerThread rst1 = startFakeHostnameRS("127.0.0.1");
+ JVMClusterUtil.RegionServerThread rst2 = startFakeHostnameRS("localhost");
+ Address addr2 = addressOf(rst2);
+ TEST_UTIL.waitFor(WAIT_TIMEOUT,
+ () -> rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().contains(addr2));
+
+ rsGroupAdmin.addRSGroup(groupName);
+ setRegex(groupName, "127\\.0\\.0\\.1|localhost");
+ try {
+ stopFakeRegionServer(rst1);
+ JVMClusterUtil.RegionServerThread rst1Restarted = startFakeHostnameRS("127.0.0.1");
+ Address addr1Restarted = addressOf(rst1Restarted);
+
+ TEST_UTIL.waitFor(WAIT_TIMEOUT,
+ () -> rsGroupAdmin.getRSGroupInfo(groupName).getServers().contains(addr1Restarted));
+ TEST_UTIL.waitFor(WAIT_TIMEOUT,
+ () -> rsGroupAdmin.getRSGroupInfo(groupName).getServers().contains(addr2));
+
+ ServerName sn1 = getServerName(addr1Restarted);
+ ServerName sn2 = getServerName(addr2);
+
+ TEST_UTIL.createMultiRegionTable(tableName, Bytes.toBytes("f"), 6);
+ TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
+ // moveTables synchronously relocates every region of a table with no RSGroup-scoped
+ // descriptor yet onto one of the target group's servers (moveTableRegionsToGroup blocks on
+ // the move futures before returning), so confinement to {sn1, sn2} is already guaranteed
+ // once this call returns -- no separate balance step is needed.
+ rsGroupAdmin.moveTables(Sets.newHashSet(tableName), groupName);
+ Map<ServerName, List<String>> perServerAfterMove = getTableServerRegionMap().get(tableName);
+ int totalAfterMove = 0;
+ for (Map.Entry<ServerName, List<String>> entry : perServerAfterMove.entrySet()) {
+ assertTrue(entry.getKey().equals(sn1) || entry.getKey().equals(sn2));
+ totalAfterMove += entry.getValue().size();
+ }
+ assertEquals(6, totalAfterMove);
+
+ // Bidirectional isolation: the group's own members must host nothing but this table.
+ for (RegionInfo region : admin.getRegions(sn1)) {
+ assertEquals(tableName, region.getTable());
+ }
+ for (RegionInfo region : admin.getRegions(sn2)) {
+ assertEquals(tableName, region.getTable());
+ }
+ } finally {
+ removeRegexGroup(groupName);
+ }
+ }
+
+ @Test
+ public void testUngracefulCrashOfRegexMemberReassignsRegionsOnlyToOtherGroupMembers()
+ throws Exception {
+ // ServerCrashProcedure recovers a dead server's regions by default with
+ // hbase.master.scp.retain.assignment=false, i.e. forceNewPlan=true for every ASSIGN TRSP it
+ // creates -- which routes through roundRobinAssignment rather than retainAssignment (see
+ // ServerCrashProcedure#assignRegions). This test's job is to confirm
+ // RSGroupBasedLoadBalancer#roundRobinAssignment still confines the
+ // crashed regex member's regions to the *other* online members of its own group -- never to
+ // default or an unrelated group -- rather than trusting the RSGroup-partitioning logic to
+ // "just work". It also asserts, via the test-only call-tracking flags, exactly which
+ // assignment methods actually ran for each step: randomAssignment + retainAssignment during
+ // moveTables, and roundRobinAssignment (not retainAssignment) during crash recovery.
+ String groupName = getGroupName("crashroundrobin");
+ rsGroupAdmin.addRSGroup(groupName);
+ // "127.0.0.1" and "localhost" are the only two distinct, real, independently-reachable
+ // identities startFakeHostnameRS supports (its RPC bind address is always "localhost"
+ // regardless of advertised hostname).
+ setRegex(groupName, "127\\.0\\.0\\.1|localhost");
+ // A second, unrelated RSGroup holding one server borrowed from default -- proves crash
+ // recovery's per-group candidate filtering excludes every other group, not just default.
+ String otherGroupName = getGroupName("crashroundrobinother");
+ RSGroupInfo otherGroupInfo = addGroup(otherGroupName, 1);
+ ServerName otherGroupServer = getServerName(otherGroupInfo.getServers().iterator().next());
+ RSGroupBasedLoadBalancer balancer = (RSGroupBasedLoadBalancer) master.getLoadBalancer();
+ try {
+ JVMClusterUtil.RegionServerThread rst1 = startFakeHostnameRS("127.0.0.1");
+ Address addr1 = addressOf(rst1);
+ TEST_UTIL.waitFor(WAIT_TIMEOUT,
+ () -> rsGroupAdmin.getRSGroupInfo(groupName).getServers().contains(addr1));
+ ServerName sn1 = getServerName(addr1);
+
+ JVMClusterUtil.RegionServerThread rst2 = startFakeHostnameRS("localhost");
+ Address addr2 = addressOf(rst2);
+ TEST_UTIL.waitFor(WAIT_TIMEOUT,
+ () -> rsGroupAdmin.getRSGroupInfo(groupName).getServers().contains(addr2));
+ ServerName sn2 = getServerName(addr2);
+
+ TEST_UTIL.createMultiRegionTable(tableName, Bytes.toBytes("f"), 6);
+ TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
+ // moveTables synchronously relocates every region of a table with no RSGroup-scoped
+ // descriptor yet onto the target group's servers (moveTableRegionsToGroup blocks on the
+ // move futures before returning), so confinement to {sn1, sn2} is already guaranteed once
+ // this call returns.
+ balancer.resetAssignmentCallFlagsForTest();
+ rsGroupAdmin.moveTables(Sets.newHashSet(tableName), groupName);
+ // moveTableRegionsToGroup picks each region's destination via randomAssignment, then moves
+ // it there via a TRSP with a non-null target (forceNewPlan=false), which the
+ // AssignmentManager
+ // queue routes through retainAssignment to commit.
+ assertTrue(balancer.isRandomAssignmentInvoked());
+ assertTrue(balancer.isRetainAssignmentInvoked());
+ assertFalse(balancer.isRoundRobinAssignmentInvoked());
+ Map<ServerName, List<String>> perServerBeforeCrash = getTableServerRegionMap().get(tableName);
+ int totalBeforeCrash = 0;
+ for (Map.Entry<ServerName, List<String>> entry : perServerBeforeCrash.entrySet()) {
+ assertTrue(entry.getKey().equals(sn1) || entry.getKey().equals(sn2));
+ totalBeforeCrash += entry.getValue().size();
+ }
+ assertEquals(6, totalBeforeCrash);
+
+ // retainAssignment's random fallback (BaseLoadBalancer#randomAssignment) places each of
+ // these regions independently, so it is possible -- roughly 3% of runs -- for all 6 to land
+ // on one of the two servers alone. Crash whichever server actually has regions, so
+ // ServerCrashProcedure always has at least one region to reassign and roundRobinAssignment
+ // is guaranteed to fire; default to the usual sn1-crashes/sn2-survives case otherwise.
+ boolean sn1HasNoRegions =
+ !perServerBeforeCrash.containsKey(sn1) || perServerBeforeCrash.get(sn1).isEmpty();
+ JVMClusterUtil.RegionServerThread rstToCrash = sn1HasNoRegions ? rst2 : rst1;
+ ServerName survivor = sn1HasNoRegions ? sn1 : sn2;
+
+ // Simulate an ungraceful crash
+ balancer.resetAssignmentCallFlagsForTest();
+ killFakeRegionServer(rstToCrash);
+
+ TEST_UTIL.waitFor(WAIT_TIMEOUT, () -> {
+ Map<ServerName, List<String>> perServer = getTableServerRegionMap().get(tableName);
+ if (perServer == null) {
+ return false;
+ }
+ int total = 0;
+ for (Map.Entry<ServerName, List<String>> entry : perServer.entrySet()) {
+ // Every region must have landed on the group's sole remaining member -- never on
+ // default or any other group, and never back on the now-dead server.
+ if (!entry.getKey().equals(survivor)) {
+ return false;
+ }
+ total += entry.getValue().size();
+ }
+ return total == 6;
+ });
+ // Confirms the crash-recovery ASSIGN TRSPs went through roundRobinAssignment, not
+ // retainAssignment -- expected since hbase.master.scp.retain.assignment defaults to false.
+ assertTrue(balancer.isRoundRobinAssignmentInvoked());
+ assertFalse(balancer.isRetainAssignmentInvoked());
+
+ // The surviving group member must host nothing but this table's regions.
+ for (RegionInfo region : admin.getRegions(survivor)) {
+ assertEquals(tableName, region.getTable());
+ }
+ // The unrelated group's own member must not have received any of the crashed member's
+ // regions either -- recovery must stay confined to the crashed server's own group.
+ for (RegionInfo region : admin.getRegions(otherGroupServer)) {
+ assertFalse(region.getTable().equals(tableName));
+ }
+ } finally {
+ removeRegexGroup(groupName);
+ removeGroup(otherGroupName);
+ }
+ }
+
+ @Test
+ public void testFallbackWhenAllRegexMembersCrash() throws Exception {
+ // RSGroupBasedLoadBalancer#generateGroupAssignments only ever looks at a group's currently
+ // online configured servers, regardless of how they became members -- regex-driven or
+ // admin-driven membership is indistinguishable to it. This confirms the
+ // hbase.rsgroup.fallback.enable=true fallback-to-default path works the same way when the
+ // group that loses every one of its servers is regex-governed.
+ String groupName = getGroupName("fallbackregex");
+ rsGroupAdmin.addRSGroup(groupName);
+ // "127.0.0.1" and "localhost" are the only two distinct, real, independently-reachable
+ // identities startFakeHostnameRS supports (its RPC bind address is always "localhost"
+ // regardless of advertised hostname).
+ setRegex(groupName, "127\\.0\\.0\\.1|localhost");
+ RSGroupBasedLoadBalancer balancer = (RSGroupBasedLoadBalancer) master.getLoadBalancer();
+ balancer.setFallbackEnabledForTest(true);
+ try {
+ JVMClusterUtil.RegionServerThread rst1 = startFakeHostnameRS("127.0.0.1");
+ Address addr1 = addressOf(rst1);
+ TEST_UTIL.waitFor(WAIT_TIMEOUT,
+ () -> rsGroupAdmin.getRSGroupInfo(groupName).getServers().contains(addr1));
+ ServerName sn1 = getServerName(addr1);
+
+ JVMClusterUtil.RegionServerThread rst2 = startFakeHostnameRS("localhost");
+ Address addr2 = addressOf(rst2);
+ TEST_UTIL.waitFor(WAIT_TIMEOUT,
+ () -> rsGroupAdmin.getRSGroupInfo(groupName).getServers().contains(addr2));
+ ServerName sn2 = getServerName(addr2);
+
+ TEST_UTIL.createMultiRegionTable(tableName, Bytes.toBytes("f"), 6);
+ TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
+ // moveTables synchronously relocates every region of a table with no RSGroup-scoped
+ // descriptor yet onto the target group's servers (moveTableRegionsToGroup blocks on the
+ // move futures before returning), so confinement to {sn1, sn2} is already guaranteed once
+ // this call returns.
+ rsGroupAdmin.moveTables(Sets.newHashSet(tableName), groupName);
+
+ // Both regex members must be hosting regions of the table before either is killed.
+ Map<ServerName, List<String>> perServerBeforeCrash = getTableServerRegionMap().get(tableName);
+ int totalBeforeCrash = 0;
+ for (ServerName member : new ServerName[] { sn1, sn2 }) {
+ List<String> regionsOnMember = perServerBeforeCrash.get(member);
+ assertFalse(regionsOnMember == null || regionsOnMember.isEmpty());
+ totalBeforeCrash += regionsOnMember.size();
+ }
+ assertEquals(6, totalBeforeCrash);
+
+ // Kill both regex members -- the group is left with zero online servers. Unlike the base
+ // cluster's servers (which remain in default throughout this test), these two fake RS were
+ // never part of default, so default keeps its original online servers and the fallback
+ // path lands there directly, with no need for a second-level fallback.
+ killFakeRegionServer(rst1);
+ killFakeRegionServer(rst2);
+
+ TEST_UTIL.waitFor(WAIT_TIMEOUT, () -> {
+ Map<ServerName, List<String>> perServer = getTableServerRegionMap().get(tableName);
+ if (perServer == null) {
+ return false;
+ }
+ RSGroupInfo defaultGroup;
+ try {
+ defaultGroup = rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP);
+ } catch (IOException e) {
+ return false;
+ }
+ int total = 0;
+ for (Map.Entry<ServerName, List<String>> entry : perServer.entrySet()) {
+ // Every region must have landed on a server that is currently in default -- never left
+ // stranded on the now-dead sn1/sn2, and never on any unrelated group.
+ if (!defaultGroup.getServers().contains(entry.getKey().getAddress())) {
+ return false;
+ }
+ total += entry.getValue().size();
+ }
+ return total == 6;
+ });
+ } finally {
+ balancer.setFallbackEnabledForTest(false);
+ removeRegexGroup(groupName);
+ }
+ }
+}