You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zg...@apache.org on 2019/06/02 14:12:18 UTC
[hbase] branch branch-2.1 updated (c5bf670 -> 7be3139)
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a change to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/hbase.git.
from c5bf670 HBASE-22511 Add some more rs-status paths to RS UI links
new b64fdd9 HBASE-22523 Refactor RegionStates#getAssignmentsByTable to make it easy to understand
new 7be3139 HBASE-22522 The integration test in master branch's nightly job has error "ERROR: Only found 1050 rows."
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../hbase_nightly_pseudo-distributed-test.sh | 2 +-
.../org/apache/hadoop/hbase/master/HMaster.java | 21 ++++---
.../hbase/master/assignment/RegionStates.java | 73 +++++++++-------------
3 files changed, 41 insertions(+), 55 deletions(-)
[hbase] 02/02: HBASE-22522 The integration test in master branch's
nightly job has error "ERROR: Only found 1050 rows."
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 7be313978e40c152aaef50f124ac090e6fbc0aeb
Author: Guanghao Zhang <zg...@apache.org>
AuthorDate: Sun Jun 2 12:00:58 2019 +0800
HBASE-22522 The integration test in master branch's nightly job has error "ERROR: Only found 1050 rows."
---
dev-support/hbase_nightly_pseudo-distributed-test.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/dev-support/hbase_nightly_pseudo-distributed-test.sh b/dev-support/hbase_nightly_pseudo-distributed-test.sh
index 60600b8..b4629a7 100755
--- a/dev-support/hbase_nightly_pseudo-distributed-test.sh
+++ b/dev-support/hbase_nightly_pseudo-distributed-test.sh
@@ -518,7 +518,7 @@ EOF
echo "Verifying row count from example."
example_rowcount=$(echo 'count "test:example"' | "${hbase_client}/bin/hbase" --config "${working_dir}/hbase-conf/" shell --noninteractive 2>/dev/null | tail -n 1)
if [ "${example_rowcount}" -gt "1050" ]; then
- echo "Found ${example_rowcount} rows, which is enough to cover 48 for import, 1000 example's use of user table regions, 2 for example's use of meta/root regions, and 1 for example's count record"
+ echo "Found ${example_rowcount} rows, which is enough to cover 48 for import, 1000 example's use of user table regions, 2 for example's use of meta/namespace regions, and 1 for example's count record"
else
echo "ERROR: Only found ${example_rowcount} rows."
fi
[hbase] 01/02: HBASE-22523 Refactor
RegionStates#getAssignmentsByTable to make it easy to understand
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit b64fdd956c4e334d4e8b4a4b145ff9871d15c04f
Author: Guanghao <zg...@apache.org>
AuthorDate: Sun Jun 2 21:21:26 2019 +0800
HBASE-22523 Refactor RegionStates#getAssignmentsByTable to make it easy to understand
---
.../org/apache/hadoop/hbase/master/HMaster.java | 21 ++++---
.../hbase/master/assignment/RegionStates.java | 73 +++++++++-------------
2 files changed, 40 insertions(+), 54 deletions(-)
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 58fd5df..63a7374 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -1648,21 +1648,22 @@ public class HMaster extends HRegionServer implements MasterServices {
}
boolean isByTable = getConfiguration().getBoolean("hbase.master.loadbalance.bytable", false);
- Map<TableName, Map<ServerName, List<RegionInfo>>> assignmentsByTable =
- this.assignmentManager.getRegionStates().getAssignmentsByTable(!isByTable);
-
- List<RegionPlan> plans = new ArrayList<>();
+ Map<TableName, Map<ServerName, List<RegionInfo>>> assignments =
+ this.assignmentManager.getRegionStates().getAssignmentsForBalancer(isByTable);
+ for (Map<ServerName, List<RegionInfo>> serverMap : assignments.values()) {
+ serverMap.keySet().removeAll(this.serverManager.getDrainingServersList());
+ }
//Give the balancer the current cluster state.
this.balancer.setClusterMetrics(getClusterMetricsWithoutCoprocessor());
- this.balancer.setClusterLoad(assignmentsByTable);
+ this.balancer.setClusterLoad(assignments);
- for (Map<ServerName, List<RegionInfo>> serverMap : assignmentsByTable.values()) {
- serverMap.keySet().removeAll(this.serverManager.getDrainingServersList());
- }
- for (Entry<TableName, Map<ServerName, List<RegionInfo>>> e : assignmentsByTable.entrySet()) {
+ List<RegionPlan> plans = new ArrayList<>();
+ for (Entry<TableName, Map<ServerName, List<RegionInfo>>> e : assignments.entrySet()) {
List<RegionPlan> partialPlans = this.balancer.balanceCluster(e.getKey(), e.getValue());
- if (partialPlans != null) plans.addAll(partialPlans);
+ if (partialPlans != null) {
+ plans.addAll(partialPlans);
+ }
}
long balanceStartTime = System.currentTimeMillis();
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStates.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStates.java
index 8f6a232..2c26f7c 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStates.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStates.java
@@ -896,55 +896,40 @@ public class RegionStates {
* Can't let out original since it can change and at least the load balancer
* wants to iterate this exported list. We need to synchronize on regions
* since all access to this.servers is under a lock on this.regions.
- * @param forceByCluster a flag to force to aggregate the server-load to the cluster level
- * @return A clone of current assignments by table.
+ *
+ * @param isByTable If <code>true</code>, return the assignments by table. If <code>false</code>,
+ * return the assignments which aggregate the server-load to the cluster level.
+ * @return A clone of current assignments.
*/
- public Map<TableName, Map<ServerName, List<RegionInfo>>> getAssignmentsByTable(
- final boolean forceByCluster) {
- if (!forceByCluster) return getAssignmentsByTable();
-
- final HashMap<ServerName, List<RegionInfo>> ensemble =
- new HashMap<ServerName, List<RegionInfo>>(serverMap.size());
- for (ServerStateNode serverNode: serverMap.values()) {
- ensemble.put(serverNode.getServerName(), serverNode.getRegionInfoList());
- }
-
- // TODO: can we use Collections.singletonMap(HConstants.ENSEMBLE_TABLE_NAME, ensemble)?
- final Map<TableName, Map<ServerName, List<RegionInfo>>> result =
- new HashMap<TableName, Map<ServerName, List<RegionInfo>>>(1);
- result.put(HConstants.ENSEMBLE_TABLE_NAME, ensemble);
- return result;
- }
-
- public Map<TableName, Map<ServerName, List<RegionInfo>>> getAssignmentsByTable() {
+ public Map<TableName, Map<ServerName, List<RegionInfo>>> getAssignmentsForBalancer(
+ boolean isByTable) {
final Map<TableName, Map<ServerName, List<RegionInfo>>> result = new HashMap<>();
- for (RegionStateNode node: regionsMap.values()) {
- Map<ServerName, List<RegionInfo>> tableResult = result.get(node.getTable());
- if (tableResult == null) {
- tableResult = new HashMap<ServerName, List<RegionInfo>>();
- result.put(node.getTable(), tableResult);
+ if (isByTable) {
+ for (RegionStateNode node : regionsMap.values()) {
+ Map<ServerName, List<RegionInfo>> tableResult =
+ result.computeIfAbsent(node.getTable(), t -> new HashMap<>());
+ final ServerName serverName = node.getRegionLocation();
+ if (serverName == null) {
+ LOG.info("Skipping, no server for " + node);
+ continue;
+ }
+ List<RegionInfo> serverResult =
+ tableResult.computeIfAbsent(serverName, s -> new ArrayList<>());
+ serverResult.add(node.getRegionInfo());
}
-
- final ServerName serverName = node.getRegionLocation();
- if (serverName == null) {
- LOG.info("Skipping, no server for " + node);
- continue;
+ // Add online servers with no assignment for the table.
+ for (Map<ServerName, List<RegionInfo>> table : result.values()) {
+ for (ServerName serverName : serverMap.keySet()) {
+ table.putIfAbsent(serverName, new ArrayList<>());
+ }
}
- List<RegionInfo> serverResult = tableResult.get(serverName);
- if (serverResult == null) {
- serverResult = new ArrayList<RegionInfo>();
- tableResult.put(serverName, serverResult);
+ } else {
+ final HashMap<ServerName, List<RegionInfo>> ensemble = new HashMap<>(serverMap.size());
+ for (ServerStateNode serverNode : serverMap.values()) {
+ ensemble.put(serverNode.getServerName(), serverNode.getRegionInfoList());
}
-
- serverResult.add(node.getRegionInfo());
- }
- // Add online servers with no assignment for the table.
- for (Map<ServerName, List<RegionInfo>> table: result.values()) {
- for (ServerName svr : serverMap.keySet()) {
- if (!table.containsKey(svr)) {
- table.put(svr, new ArrayList<RegionInfo>());
- }
- }
+ // Use a fake table name to represent the whole cluster's assignments
+ result.put(HConstants.ENSEMBLE_TABLE_NAME, ensemble);
}
return result;
}