You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by vj...@apache.org on 2020/07/01 14:57:50 UTC
[hbase] branch master updated: HBASE-24560 Add a new option of
designatedfile in RegionMover
This is an automated email from the ASF dual-hosted git repository.
vjasani pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new afe2eac HBASE-24560 Add a new option of designatedfile in RegionMover
afe2eac is described below
commit afe2eac7b6c92e7be6067e5dfaa69e076679770c
Author: Baiqiang Zhao <zb...@gmail.com>
AuthorDate: Wed Jul 1 20:20:10 2020 +0530
HBASE-24560 Add a new option of designatedfile in RegionMover
Closes #1901
Signed-off-by: Anoop Sam John <an...@apache.org>
Signed-off-by: Viraj Jasani <vj...@apache.org>
---
.../org/apache/hadoop/hbase/util/RegionMover.java | 80 ++++++++++++++++------
.../apache/hadoop/hbase/util/TestRegionMover.java | 79 +++++++++++++++++++++
2 files changed, 137 insertions(+), 22 deletions(-)
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java
index 742f25a..693bfc5 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java
@@ -101,6 +101,7 @@ public class RegionMover extends AbstractHBaseTool implements Closeable {
private String hostname;
private String filename;
private String excludeFile;
+ private String designatedFile;
private int port;
private Connection conn;
private Admin admin;
@@ -109,6 +110,7 @@ public class RegionMover extends AbstractHBaseTool implements Closeable {
this.hostname = builder.hostname;
this.filename = builder.filename;
this.excludeFile = builder.excludeFile;
+ this.designatedFile = builder.designatedFile;
this.maxthreads = builder.maxthreads;
this.ack = builder.ack;
this.port = builder.port;
@@ -130,7 +132,8 @@ public class RegionMover extends AbstractHBaseTool implements Closeable {
/**
* Builder for Region mover. Use the {@link #build()} method to create RegionMover object. Has
* {@link #filename(String)}, {@link #excludeFile(String)}, {@link #maxthreads(int)},
- * {@link #ack(boolean)}, {@link #timeout(int)} methods to set the corresponding options
+ * {@link #ack(boolean)}, {@link #timeout(int)}, {@link #designatedFile(String)} methods to set
+ * the corresponding options.
*/
public static class RegionMoverBuilder {
private boolean ack = true;
@@ -139,6 +142,7 @@ public class RegionMover extends AbstractHBaseTool implements Closeable {
private String hostname;
private String filename;
private String excludeFile = null;
+ private String designatedFile = null;
private String defaultDir = System.getProperty("java.io.tmpdir");
@VisibleForTesting
final int port;
@@ -206,6 +210,18 @@ public class RegionMover extends AbstractHBaseTool implements Closeable {
}
/**
+ * Set the designated file. Designated file contains hostnames where region moves. Designated
+ * file should have 'host:port' per line. Port is mandatory here as we can have many RS running
+ * on a single host.
+ * @param designatedFile The designated file
+ * @return RegionMoverBuilder object
+ */
+ public RegionMoverBuilder designatedFile(String designatedFile) {
+ this.designatedFile = designatedFile;
+ return this;
+ }
+
+ /**
* Set ack/noAck mode.
* <p>
* In ack mode regions are acknowledged before and after moving and the move is retried
@@ -413,7 +429,8 @@ public class RegionMover extends AbstractHBaseTool implements Closeable {
* Unload regions from given {@link #hostname} using ack/noAck mode and {@link #maxthreads}.In
* noAck mode we do not make sure that region is successfully online on the target region
* server,hence it is best effort.We do not unload regions to hostnames given in
- * {@link #excludeFile}.
+ * {@link #excludeFile}. If designatedFile is present with some contents, we will unload regions
+ * to hostnames provided in {@link #designatedFile}
* @return true if unloading succeeded, false otherwise
*/
public boolean unload() throws InterruptedException, ExecutionException, TimeoutException {
@@ -435,8 +452,11 @@ public class RegionMover extends AbstractHBaseTool implements Closeable {
LOG.debug("List of region servers: {}", regionServers);
return false;
}
+ // Remove RS not present in the designated file
+ includeExcludeRegionServers(designatedFile, regionServers, true);
+
// Remove RS present in the exclude file
- stripExcludes(regionServers);
+ includeExcludeRegionServers(excludeFile, regionServers, false);
// Remove decommissioned RS
Set<ServerName> decommissionedRS = new HashSet<>(admin.listDecommissionedRegionServers());
@@ -653,41 +673,52 @@ public class RegionMover extends AbstractHBaseTool implements Closeable {
}
/**
- * @return List of servers from the exclude file in format 'hostname:port'.
+ * @param filename The file should have 'host:port' per line
+ * @return List of servers from the file in format 'hostname:port'.
*/
- private List<String> readExcludes(String excludeFile) throws IOException {
- List<String> excludeServers = new ArrayList<>();
- if (excludeFile == null) {
- return excludeServers;
- } else {
+ private List<String> readServersFromFile(String filename) throws IOException {
+ List<String> servers = new ArrayList<>();
+ if (filename != null) {
try {
- Files.readAllLines(Paths.get(excludeFile)).stream().map(String::trim)
- .filter(((Predicate<String>) String::isEmpty).negate()).map(String::toLowerCase)
- .forEach(excludeServers::add);
+ Files.readAllLines(Paths.get(filename)).stream().map(String::trim)
+ .filter(((Predicate<String>) String::isEmpty).negate()).map(String::toLowerCase)
+ .forEach(servers::add);
} catch (IOException e) {
- LOG.warn("Exception while reading excludes file, continuing anyways", e);
+ LOG.error("Exception while reading servers from file,", e);
+ throw e;
}
- return excludeServers;
}
+ return servers;
}
/**
- * Excludes the servername whose hostname and port portion matches the list given in exclude file
+ * Designates or excludes the servername whose hostname and port portion matches the list given
+ * in the file.
+ * Example:<br>
+ * If you want to designated RSs, suppose designatedFile has RS1, regionServers has RS1, RS2 and
+ * RS3. When we call includeExcludeRegionServers(designatedFile, regionServers, true), RS2 and
+ * RS3 are removed from regionServers list so that regions can move to only RS1.
+ * If you want to exclude RSs, suppose excludeFile has RS1, regionServers has RS1, RS2 and RS3.
+ * When we call includeExcludeRegionServers(excludeFile, servers, false), RS1 is removed from
+ * regionServers list so that regions can move to only RS2 and RS3.
*/
- private void stripExcludes(List<ServerName> regionServers) throws IOException {
- if (excludeFile != null) {
- List<String> excludes = readExcludes(excludeFile);
+ private void includeExcludeRegionServers(String fileName, List<ServerName> regionServers,
+ boolean isInclude) throws IOException {
+ if (fileName != null) {
+ List<String> servers = readServersFromFile(fileName);
+ if (servers.isEmpty()) {
+ LOG.warn("No servers provided in the file: {}." + fileName);
+ return;
+ }
Iterator<ServerName> i = regionServers.iterator();
while (i.hasNext()) {
String rs = i.next().getServerName();
String rsPort = rs.split(ServerName.SERVERNAME_SEPARATOR)[0].toLowerCase() + ":" + rs
- .split(ServerName.SERVERNAME_SEPARATOR)[1];
- if (excludes.contains(rsPort)) {
+ .split(ServerName.SERVERNAME_SEPARATOR)[1];
+ if (isInclude != servers.contains(rsPort)) {
i.remove();
}
}
- LOG.info("Valid Region server targets are:" + regionServers.toString());
- LOG.info("Excluded Servers are" + excludes.toString());
}
}
@@ -773,6 +804,8 @@ public class RegionMover extends AbstractHBaseTool implements Closeable {
this.addOptWithArg("x", "excludefile",
"File with <hostname:port> per line to exclude as unload targets; default excludes only "
+ "target host; useful for rack decommisioning.");
+ this.addOptWithArg("d","designatedfile","File with <hostname:port> per line as unload targets;"
+ + "default is all online hosts");
this.addOptWithArg("f", "filename",
"File to save regions list into unloading, or read from loading; "
+ "default /tmp/<usernamehostname:port>");
@@ -801,6 +834,9 @@ public class RegionMover extends AbstractHBaseTool implements Closeable {
if (cmd.hasOption('x')) {
rmbuilder.excludeFile(cmd.getOptionValue('x'));
}
+ if (cmd.hasOption('d')) {
+ rmbuilder.designatedFile(cmd.getOptionValue('d'));
+ }
if (cmd.hasOption('t')) {
rmbuilder.timeout(Integer.parseInt(cmd.getOptionValue('t')));
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionMover.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionMover.java
index 4d18f4b..b7e947c 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionMover.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionMover.java
@@ -181,6 +181,85 @@ public class TestRegionMover {
}
@Test
+ public void testDesignatedFile() throws Exception{
+ MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
+ File designatedFile = new File(TEST_UTIL.getDataTestDir().toUri().getPath(),
+ "designated_file");
+ HRegionServer designatedServer = cluster.getRegionServer(0);
+ try(FileWriter fos = new FileWriter(designatedFile)) {
+ String designatedHostname = designatedServer.getServerName().getHostname();
+ int designatedServerPort = designatedServer.getServerName().getPort();
+ String excludeServerName = designatedHostname + ":" + designatedServerPort;
+ fos.write(excludeServerName);
+ }
+ int regionsInDesignatedServer = designatedServer.getNumberOfOnlineRegions();
+ HRegionServer regionServer = cluster.getRegionServer(1);
+ String rsName = regionServer.getServerName().getHostname();
+ int port = regionServer.getServerName().getPort();
+ String rs = rsName + ":" + port;
+ int regionsInRegionServer = regionServer.getNumberOfOnlineRegions();
+ RegionMoverBuilder rmBuilder = new RegionMoverBuilder(rs, TEST_UTIL.getConfiguration())
+ .designatedFile(designatedFile.getCanonicalPath());
+ try (RegionMover rm = rmBuilder.build()) {
+ LOG.debug("Unloading {} regions", rs);
+ rm.unload();
+ assertEquals(0, regionServer.getNumberOfOnlineRegions());
+ assertEquals(regionsInDesignatedServer + regionsInRegionServer,
+ designatedServer.getNumberOfOnlineRegions());
+ LOG.debug("Before:{} After:{}", regionsInDesignatedServer,
+ designatedServer.getNumberOfOnlineRegions());
+ }
+ }
+
+ @Test
+ public void testExcludeAndDesignated() throws Exception{
+ MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
+ // create designated file
+ File designatedFile = new File(TEST_UTIL.getDataTestDir().toUri().getPath(),
+ "designated_file");
+ HRegionServer designatedServer = cluster.getRegionServer(0);
+ try(FileWriter fos = new FileWriter(designatedFile)) {
+ String designatedHostname = designatedServer.getServerName().getHostname();
+ int designatedServerPort = designatedServer.getServerName().getPort();
+ String excludeServerName = designatedHostname + ":" + designatedServerPort;
+ fos.write(excludeServerName);
+ }
+ int regionsInDesignatedServer = designatedServer.getNumberOfOnlineRegions();
+ // create exclude file
+ File excludeFile = new File(TEST_UTIL.getDataTestDir().toUri().getPath(), "exclude_file");
+ HRegionServer excludeServer = cluster.getRegionServer(1);
+ try(FileWriter fos = new FileWriter(excludeFile)) {
+ String excludeHostname = excludeServer.getServerName().getHostname();
+ int excludeServerPort = excludeServer.getServerName().getPort();
+ String excludeServerName = excludeHostname + ":" + excludeServerPort;
+ fos.write(excludeServerName);
+ }
+ int regionsInExcludeServer = excludeServer.getNumberOfOnlineRegions();
+
+ HRegionServer targetRegionServer = cluster.getRegionServer(2);
+ String rsName = targetRegionServer.getServerName().getHostname();
+ int port = targetRegionServer.getServerName().getPort();
+ String rs = rsName + ":" + port;
+ int regionsInTargetRegionServer = targetRegionServer.getNumberOfOnlineRegions();
+
+ RegionMoverBuilder rmBuilder = new RegionMoverBuilder(rs, TEST_UTIL.getConfiguration())
+ .designatedFile(designatedFile.getCanonicalPath())
+ .excludeFile(excludeFile.getCanonicalPath());
+ try (RegionMover rm = rmBuilder.build()) {
+ LOG.debug("Unloading {}", rs);
+ rm.unload();
+ assertEquals(0, targetRegionServer.getNumberOfOnlineRegions());
+ assertEquals(regionsInDesignatedServer + regionsInTargetRegionServer,
+ designatedServer.getNumberOfOnlineRegions());
+ LOG.debug("DesignatedServer Before:{} After:{}", regionsInDesignatedServer,
+ designatedServer.getNumberOfOnlineRegions());
+ assertEquals(regionsInExcludeServer, excludeServer.getNumberOfOnlineRegions());
+ LOG.debug("ExcludeServer Before:{} After:{}", regionsInExcludeServer,
+ excludeServer.getNumberOfOnlineRegions());
+ }
+ }
+
+ @Test
public void testRegionServerPort() {
MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
HRegionServer regionServer = cluster.getRegionServer(0);