You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zh...@apache.org on 2019/11/05 06:35:00 UTC
[hbase] 06/14: HBASE-22820 Do not need to persist default rs group
now (#482)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch HBASE-22514
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 935f0ac41b12b5e81ffd1238c037596fc9108daf
Author: linkaline <li...@gmail.com>
AuthorDate: Fri Aug 16 16:52:41 2019 +0800
HBASE-22820 Do not need to persist default rs group now (#482)
Signed-off-by: Duo Zhang <zh...@apache.org>
---
.../hbase/rsgroup/RSGroupInfoManagerImpl.java | 40 +++++++++++++---------
.../hbase/rsgroup/VerifyingRSGroupAdminClient.java | 25 ++++++++++++++
2 files changed, 49 insertions(+), 16 deletions(-)
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java
index 37f3ce6..eaf23f3 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java
@@ -396,11 +396,13 @@ final class RSGroupInfoManagerImpl implements RSGroupInfoManager {
}
// populate puts
- for (RSGroupInfo RSGroupInfo : groupMap.values()) {
- RSGroupProtos.RSGroupInfo proto = ProtobufUtil.toProtoGroupInfo(RSGroupInfo);
- Put p = new Put(Bytes.toBytes(RSGroupInfo.getName()));
- p.addColumn(META_FAMILY_BYTES, META_QUALIFIER_BYTES, proto.toByteArray());
- mutations.add(p);
+ for (RSGroupInfo gi : groupMap.values()) {
+ if (!gi.getName().equals(RSGroupInfo.DEFAULT_GROUP)) {
+ RSGroupProtos.RSGroupInfo proto = ProtobufUtil.toProtoGroupInfo(gi);
+ Put p = new Put(Bytes.toBytes(gi.getName()));
+ p.addColumn(META_FAMILY_BYTES, META_QUALIFIER_BYTES, proto.toByteArray());
+ mutations.add(p);
+ }
}
if (mutations.size() > 0) {
@@ -449,7 +451,12 @@ final class RSGroupInfoManagerImpl implements RSGroupInfoManager {
// Make changes visible after having been persisted to the source of truth
resetRSGroupMap(newGroupMap);
+ saveRSGroupMapToZK(newGroupMap);
+ updateCacheOfRSGroups(newGroupMap.keySet());
+ }
+
+ private void saveRSGroupMapToZK(Map<String, RSGroupInfo> newGroupMap) throws IOException {
try {
String groupBasePath =
ZNodePaths.joinZNode(watcher.getZNodePaths().baseZNode, RS_GROUP_ZNODE);
@@ -463,14 +470,16 @@ final class RSGroupInfoManagerImpl implements RSGroupInfoManager {
}
}
- for (RSGroupInfo RSGroupInfo : newGroupMap.values()) {
- String znode = ZNodePaths.joinZNode(groupBasePath, RSGroupInfo.getName());
- RSGroupProtos.RSGroupInfo proto = ProtobufUtil.toProtoGroupInfo(RSGroupInfo);
- LOG.debug("Updating znode: " + znode);
- ZKUtil.createAndFailSilent(watcher, znode);
- zkOps.add(ZKUtil.ZKUtilOp.deleteNodeFailSilent(znode));
- zkOps.add(ZKUtil.ZKUtilOp.createAndFailSilent(znode,
- ProtobufUtil.prependPBMagic(proto.toByteArray())));
+ for (RSGroupInfo gi : newGroupMap.values()) {
+ if (!gi.getName().equals(RSGroupInfo.DEFAULT_GROUP)) {
+ String znode = ZNodePaths.joinZNode(groupBasePath, gi.getName());
+ RSGroupProtos.RSGroupInfo proto = ProtobufUtil.toProtoGroupInfo(gi);
+ LOG.debug("Updating znode: " + znode);
+ ZKUtil.createAndFailSilent(watcher, znode);
+ zkOps.add(ZKUtil.ZKUtilOp.deleteNodeFailSilent(znode));
+ zkOps.add(ZKUtil.ZKUtilOp.createAndFailSilent(znode,
+ ProtobufUtil.prependPBMagic(proto.toByteArray())));
+ }
}
LOG.debug("Writing ZK GroupInfo count: " + zkOps.size());
@@ -480,7 +489,6 @@ final class RSGroupInfoManagerImpl implements RSGroupInfoManager {
masterServices.abort("Failed to write to rsGroupZNode", e);
throw new IOException("Failed to write to rsGroupZNode", e);
}
- updateCacheOfRSGroups(newGroupMap.keySet());
}
/**
@@ -540,12 +548,12 @@ final class RSGroupInfoManagerImpl implements RSGroupInfoManager {
// Called by ServerEventsListenerThread. Synchronize on this because redoing
// the rsGroupMap then writing it out.
- private synchronized void updateDefaultServers(SortedSet<Address> servers) throws IOException {
+ private synchronized void updateDefaultServers(SortedSet<Address> servers) {
RSGroupInfo info = rsGroupMap.get(RSGroupInfo.DEFAULT_GROUP);
RSGroupInfo newInfo = new RSGroupInfo(info.getName(), servers);
HashMap<String, RSGroupInfo> newGroupMap = Maps.newHashMap(rsGroupMap);
newGroupMap.put(newInfo.getName(), newInfo);
- flushConfig(newGroupMap);
+ resetRSGroupMap(newGroupMap);
}
/**
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdminClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdminClient.java
index a8cd277..28131a9 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdminClient.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdminClient.java
@@ -26,8 +26,11 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.SortedSet;
+import java.util.regex.Pattern;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.NamespaceDescriptor;
+import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
@@ -133,6 +136,13 @@ public class VerifyingRSGroupAdminClient extends RSGroupAdminClient {
tds.addAll(admin.listTableDescriptors());
tds.addAll(admin.listTableDescriptorsByNamespace(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME));
}
+ SortedSet<Address> lives = Sets.newTreeSet();
+ for (ServerName sn : conn.getAdmin().getClusterMetrics().getLiveServerMetrics().keySet()) {
+ lives.add(sn.getAddress());
+ }
+ for (ServerName sn : conn.getAdmin().listDecommissionedRegionServers()) {
+ lives.remove(sn.getAddress());
+ }
try (Table table = conn.getTable(RSGroupInfoManagerImpl.RSGROUP_TABLE_NAME);
ResultScanner scanner = table.getScanner(new Scan())) {
for (;;) {
@@ -144,8 +154,22 @@ public class VerifyingRSGroupAdminClient extends RSGroupAdminClient {
RSGroupInfoManagerImpl.META_FAMILY_BYTES, RSGroupInfoManagerImpl.META_QUALIFIER_BYTES));
RSGroupInfo rsGroupInfo = ProtobufUtil.toGroupInfo(proto);
groupMap.put(proto.getName(), RSGroupUtil.fillTables(rsGroupInfo, tds));
+ for(Address address : rsGroupInfo.getServers()){
+ lives.remove(address);
+ }
}
}
+ SortedSet<TableName> tables = Sets.newTreeSet();
+ for (TableDescriptor td : conn.getAdmin().listTableDescriptors(Pattern.compile(".*"),
+ true)){
+ String groupName = td.getRegionServerGroup().orElse(RSGroupInfo.DEFAULT_GROUP);
+ if (groupName.equals(RSGroupInfo.DEFAULT_GROUP)) {
+ tables.add(td.getTableName());
+ }
+ }
+
+ groupMap.put(RSGroupInfo.DEFAULT_GROUP,
+ new RSGroupInfo(RSGroupInfo.DEFAULT_GROUP, lives, tables));
assertEquals(Sets.newHashSet(groupMap.values()), Sets.newHashSet(wrapped.listRSGroups()));
try {
String groupBasePath = ZNodePaths.joinZNode(zkw.getZNodePaths().baseZNode, "rsgroup");
@@ -160,6 +184,7 @@ public class VerifyingRSGroupAdminClient extends RSGroupAdminClient {
zList.add(RSGroupUtil.fillTables(rsGroupInfo, tds));
}
}
+ groupMap.remove(RSGroupInfo.DEFAULT_GROUP);
assertEquals(zList.size(), groupMap.size());
for (RSGroupInfo rsGroupInfo : zList) {
assertTrue(groupMap.get(rsGroupInfo.getName()).equals(rsGroupInfo));