You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2021/01/12 23:58:20 UTC
[hbase] branch branch-2.4 updated: HBASE-25211 : Rack awareness in
RegionMover (#2795)
This is an automated email from the ASF dual-hosted git repository.
apurtell pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.4 by this push:
new e8279b2 HBASE-25211 : Rack awareness in RegionMover (#2795)
e8279b2 is described below
commit e8279b26fa23e2af2808158008db49d3a69d8b43
Author: Viraj Jasani <vj...@apache.org>
AuthorDate: Wed Jan 13 05:01:26 2021 +0530
HBASE-25211 : Rack awareness in RegionMover (#2795)
Signed-off-by: Andrew Purtell <ap...@apache.org>
---
.../apache/hadoop/hbase/master/RackManager.java | 3 -
.../org/apache/hadoop/hbase/util/RegionMover.java | 66 +++++++-
.../apache/hadoop/hbase/util/TestRegionMover3.java | 188 +++++++++++++++++++++
3 files changed, 253 insertions(+), 4 deletions(-)
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RackManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RackManager.java
index 3ed2006..54ccac0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RackManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RackManager.java
@@ -22,8 +22,6 @@ import java.util.Collections;
import java.util.List;
import org.apache.yetus.audience.InterfaceAudience;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.util.ReflectionUtils;
@@ -36,7 +34,6 @@ import org.apache.hadoop.net.ScriptBasedMapping;
*/
@InterfaceAudience.Private
public class RackManager {
- private static final Logger LOG = LoggerFactory.getLogger(RackManager.class);
public static final String UNKNOWN_RACK = "Unknown Rack";
private DNSToSwitchMapping switchMapping;
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java
index 5822222..7b429d0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java
@@ -58,6 +58,7 @@ import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.DoNotRetryRegionException;
import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.master.RackManager;
import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
@@ -83,6 +84,7 @@ public class RegionMover extends AbstractHBaseTool implements Closeable {
public static final int DEFAULT_MOVE_RETRIES_MAX = 5;
public static final int DEFAULT_MOVE_WAIT_MAX = 60;
public static final int DEFAULT_SERVERSTART_WAIT_MAX = 180;
+ private final RackManager rackManager;
private static final Logger LOG = LoggerFactory.getLogger(RegionMover.class);
@@ -111,9 +113,16 @@ public class RegionMover extends AbstractHBaseTool implements Closeable {
setConf(builder.conf);
this.conn = ConnectionFactory.createConnection(conf);
this.admin = conn.getAdmin();
+ // Only while running unit tests, builder.rackManager will not be null for the convenience of
+ // providing custom rackManager. Otherwise for regular workflow/user triggered action,
+ // builder.rackManager is supposed to be null. Hence, setter of builder.rackManager is
+ // provided as @InterfaceAudience.Private and it is commented that this is just
+ // to be used by unit test.
+ rackManager = builder.rackManager == null ? new RackManager(conf) : builder.rackManager;
}
private RegionMover() {
+ rackManager = new RackManager(conf);
}
@Override
@@ -140,6 +149,7 @@ public class RegionMover extends AbstractHBaseTool implements Closeable {
@InterfaceAudience.Private
final int port;
private final Configuration conf;
+ private RackManager rackManager;
public RegionMoverBuilder(String hostname) {
this(hostname, createConf());
@@ -243,6 +253,19 @@ public class RegionMover extends AbstractHBaseTool implements Closeable {
}
/**
+ * Set specific rackManager implementation.
+ * This setter method is for testing purpose only.
+ *
+ * @param rackManager rackManager impl
+ * @return RegionMoverBuilder object
+ */
+ @InterfaceAudience.Private
+ public RegionMoverBuilder rackManager(RackManager rackManager) {
+ this.rackManager = rackManager;
+ return this;
+ }
+
+ /**
* This method builds the appropriate RegionMover object which can then be used to load/unload
* using load and unload methods
* @return RegionMover object
@@ -325,9 +348,31 @@ public class RegionMover extends AbstractHBaseTool implements Closeable {
* server,hence it is best effort.We do not unload regions to hostnames given in
* {@link #excludeFile}. If designatedFile is present with some contents, we will unload regions
* to hostnames provided in {@link #designatedFile}
+ *
* @return true if unloading succeeded, false otherwise
*/
public boolean unload() throws InterruptedException, ExecutionException, TimeoutException {
+ return unloadRegions(false);
+ }
+
+ /**
+ * Unload regions from given {@link #hostname} using ack/noAck mode and {@link #maxthreads}.In
+ * noAck mode we do not make sure that region is successfully online on the target region
+ * server,hence it is best effort.We do not unload regions to hostnames given in
+ * {@link #excludeFile}. If designatedFile is present with some contents, we will unload regions
+ * to hostnames provided in {@link #designatedFile}.
+ * While unloading regions, destination RegionServers are selected from different rack i.e
+ * regions should not move to any RegionServers that belong to same rack as source RegionServer.
+ *
+ * @return true if unloading succeeded, false otherwise
+ */
+ public boolean unloadFromRack()
+ throws InterruptedException, ExecutionException, TimeoutException {
+ return unloadRegions(true);
+ }
+
+ private boolean unloadRegions(boolean unloadFromRack) throws InterruptedException,
+ ExecutionException, TimeoutException {
deleteFile(this.filename);
ExecutorService unloadPool = Executors.newFixedThreadPool(1);
Future<Boolean> unloadTask = unloadPool.submit(() -> {
@@ -350,6 +395,23 @@ public class RegionMover extends AbstractHBaseTool implements Closeable {
// Remove RS present in the exclude file
includeExcludeRegionServers(excludeFile, regionServers, false);
+ if (unloadFromRack) {
+ // remove regionServers that belong to same rack (as source host) since the goal is to
+ // unload regions from source regionServer to destination regionServers
+ // that belong to different rack only.
+ String sourceRack = rackManager.getRack(server);
+ List<String> racks = rackManager.getRack(regionServers);
+ Iterator<ServerName> iterator = regionServers.iterator();
+ int i = 0;
+ while (iterator.hasNext()) {
+ iterator.next();
+ if (racks.size() > i && racks.get(i) != null && racks.get(i).equals(sourceRack)) {
+ iterator.remove();
+ }
+ i++;
+ }
+ }
+
// Remove decommissioned RS
Set<ServerName> decommissionedRS = new HashSet<>(admin.listDecommissionedRegionServers());
if (CollectionUtils.isNotEmpty(decommissionedRS)) {
@@ -640,7 +702,7 @@ public class RegionMover extends AbstractHBaseTool implements Closeable {
@Override
protected void addOptions() {
this.addRequiredOptWithArg("r", "regionserverhost", "region server <hostname>|<hostname:port>");
- this.addRequiredOptWithArg("o", "operation", "Expected: load/unload");
+ this.addRequiredOptWithArg("o", "operation", "Expected: load/unload/unload_from_rack");
this.addOptWithArg("m", "maxthreads",
"Define the maximum number of threads to use to unload and reload the regions");
this.addOptWithArg("x", "excludefile",
@@ -693,6 +755,8 @@ public class RegionMover extends AbstractHBaseTool implements Closeable {
success = rm.load();
} else if (loadUnload.equalsIgnoreCase("unload")) {
success = rm.unload();
+ } else if (loadUnload.equalsIgnoreCase("unload_from_rack")) {
+ success = rm.unloadFromRack();
} else {
printUsage();
success = false;
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionMover3.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionMover3.java
new file mode 100644
index 0000000..1903fa6
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionMover3.java
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.util;
+
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.MiniHBaseCluster;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
+import org.apache.hadoop.hbase.master.RackManager;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.testclassification.MiscTests;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+
+@Category({ MiscTests.class, LargeTests.class})
+public class TestRegionMover3 {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestRegionMover3.class);
+
+ @Rule
+ public TestName name = new TestName();
+
+ private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+ private static ServerName rs0;
+ private static ServerName rs1;
+ private static ServerName rs2;
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ TEST_UTIL.startMiniCluster(3);
+ MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
+ rs0 = cluster.getRegionServer(0).getServerName();
+ rs1 = cluster.getRegionServer(1).getServerName();
+ rs2 = cluster.getRegionServer(2).getServerName();
+ TEST_UTIL.getAdmin().balancerSwitch(false, true);
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ TEST_UTIL.shutdownMiniCluster();
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ final TableName tableName = TableName.valueOf(name.getMethodName());
+ TableDescriptor tableDesc = TableDescriptorBuilder.newBuilder(tableName)
+ .setColumnFamily(ColumnFamilyDescriptorBuilder.of("fam1")).build();
+ int startKey = 0;
+ int endKey = 80000;
+ TEST_UTIL.getAdmin().createTable(tableDesc, Bytes.toBytes(startKey), Bytes.toBytes(endKey), 9);
+ }
+
+ @Test
+ public void testRegionUnloadWithRack() throws Exception {
+ final TableName tableName = TableName.valueOf(name.getMethodName());
+ MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
+ Admin admin = TEST_UTIL.getAdmin();
+ Table table = TEST_UTIL.getConnection().getTable(tableName);
+ List<Put> puts = IntStream.range(10, 50000)
+ .mapToObj(i -> new Put(Bytes.toBytes(i))
+ .addColumn(Bytes.toBytes("fam1"), Bytes.toBytes("q1"), Bytes.toBytes("val_" + i)))
+ .collect(Collectors.toList());
+ table.put(puts);
+ admin.flush(tableName);
+ admin.compact(tableName);
+ Thread.sleep(3000);
+ HRegionServer hRegionServer0 = cluster.getRegionServer(0);
+ HRegionServer hRegionServer1 = cluster.getRegionServer(1);
+ HRegionServer hRegionServer2 = cluster.getRegionServer(2);
+ int numRegions0 = hRegionServer0.getNumberOfOnlineRegions();
+ int numRegions1 = hRegionServer1.getNumberOfOnlineRegions();
+ int numRegions2 = hRegionServer2.getNumberOfOnlineRegions();
+
+ Assert.assertTrue(numRegions0 >= 3);
+ Assert.assertTrue(numRegions1 >= 3);
+ Assert.assertTrue(numRegions2 >= 3);
+ int totalRegions = numRegions0 + numRegions1 + numRegions2;
+
+ // source RS: rs0
+ String sourceRSName = rs0.getAddress().toString();
+
+ // move all regions from rs1 to rs0
+ for (HRegion region : hRegionServer1.getRegions()) {
+ TEST_UTIL.getAdmin().move(region.getRegionInfo().getEncodedNameAsBytes(), rs0);
+ }
+ TEST_UTIL.waitFor(5000, () -> {
+ int newNumRegions0 = hRegionServer0.getNumberOfOnlineRegions();
+ int newNumRegions1 = hRegionServer1.getNumberOfOnlineRegions();
+ return newNumRegions1 == 0 && newNumRegions0 == (numRegions0 + numRegions1);
+ });
+
+ // regionMover obj on rs0. While unloading regions from rs0
+ // with default rackManager, which resolves "/default-rack" for each server, no region
+ // is moved while using unloadFromRack() as all rs belong to same rack.
+ RegionMover.RegionMoverBuilder rmBuilder =
+ new RegionMover.RegionMoverBuilder(sourceRSName, TEST_UTIL.getConfiguration())
+ .ack(true)
+ .maxthreads(8);
+ try (RegionMover regionMover = rmBuilder.build()) {
+ regionMover.unloadFromRack();
+ int newNumRegions0 = hRegionServer0.getNumberOfOnlineRegions();
+ int newNumRegions1 = hRegionServer1.getNumberOfOnlineRegions();
+ int newNumRegions2 = hRegionServer2.getNumberOfOnlineRegions();
+ Assert.assertEquals(0, newNumRegions1);
+ Assert.assertEquals(totalRegions, newNumRegions0 + newNumRegions2);
+ }
+
+ // use custom rackManager, which resolves "rack-1" for rs0 and rs1,
+ // while "rack-2" for rs2. Hence, unloadFromRack() from rs0 should move all
+ // regions that belong to rs0 to rs2 only, and nothing should be moved to rs1
+ // as rs0 and rs1 belong to same rack.
+ rmBuilder.rackManager(new MockRackManager());
+ try (RegionMover regionMover = rmBuilder.build()) {
+ regionMover.unloadFromRack();
+ int newNumRegions0 = hRegionServer0.getNumberOfOnlineRegions();
+ int newNumRegions1 = hRegionServer1.getNumberOfOnlineRegions();
+ int newNumRegions2 = hRegionServer2.getNumberOfOnlineRegions();
+ Assert.assertEquals(0, newNumRegions0);
+ Assert.assertEquals(0, newNumRegions1);
+ Assert.assertEquals(totalRegions, newNumRegions2);
+ }
+
+ }
+
+ private static class MockRackManager extends RackManager {
+
+ private static final String RACK_2 = "rack-2";
+ private static final String RACK_1 = "rack-1";
+
+ @Override
+ public String getRack(ServerName server) {
+ return rs2.equals(server) ? RACK_2 : RACK_1;
+ }
+
+ @Override
+ public List<String> getRack(List<ServerName> servers) {
+ List<String> racks = new ArrayList<>();
+ servers.forEach(serverName -> {
+ if (rs2.equals(serverName)) {
+ racks.add(RACK_2);
+ } else {
+ racks.add(RACK_1);
+ }
+ });
+ return racks;
+ }
+ }
+
+}