You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2018/09/17 21:30:34 UTC
[hbase-operator-tools] branch master updated: Add assigns/unassigns
commands.
This is an automated email from the ASF dual-hosted git repository.
stack pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase-operator-tools.git
The following commit(s) were added to refs/heads/master by this push:
new c6af272 Add assigns/unassigns commands.
c6af272 is described below
commit c6af272f82cd70694dcc717a4b7c5c6dcefa0505
Author: Michael Stack <st...@apache.org>
AuthorDate: Mon Sep 17 14:29:32 2018 -0700
Add assigns/unassigns commands.
Tests are disabled for the moment while figure issue with shaded client.
---
hbase-hbck2/pom.xml | 8 +-
.../src/main/java/org/apache/hbase/HBCK2.java | 91 ++++++++++++++++----
.../src/test/java/org/apache/hbase/TestHBCK2.java | 96 ++++++++++++++++++++++
pom.xml | 2 +-
4 files changed, 181 insertions(+), 16 deletions(-)
diff --git a/hbase-hbck2/pom.xml b/hbase-hbck2/pom.xml
index 4996f97..a550ebe 100644
--- a/hbase-hbck2/pom.xml
+++ b/hbase-hbck2/pom.xml
@@ -114,7 +114,13 @@
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-shaded-client</artifactId>
- <version>2.1.1-SNAPSHOT</version>
+ <version>${hbase.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-testing-util</artifactId>
+ <version>${hbase.version}</version>
+ <scope>test</scope>
</dependency>
</dependencies>
diff --git a/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java b/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java
index 4234637..2d06a78 100644
--- a/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java
+++ b/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java
@@ -41,6 +41,7 @@ import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.util.Arrays;
+import java.util.List;
import java.util.stream.Collectors;
/**
@@ -49,6 +50,7 @@ import java.util.stream.Collectors;
*/
// TODO:
// + Add bulk assign/unassigns. If 60k OPENING regions, doing it via shell takes 10-60 seconds each.
+// + On assign, can we look to see if existing assign and if so fail until cancelled?
// + Add test of Master version to ensure it supports hbck functionality.
// + Doc how we just take pointer to zk ensemble... If want to do more exotic config. on client,
// then add a hbase-site.xml onto CLASSPATH for this tool to pick up.
@@ -56,11 +58,42 @@ public class HBCK2 {
private static final Logger LOG = LogManager.getLogger(HBCK2.class);
public static final int EXIT_SUCCESS = 0;
public static final int EXIT_FAILURE = 1;
+ // Commands
private static final String SET_TABLE_STATE = "setTableState";
+ private static final String ASSIGN = "assign";
+ private static final String UNASSIGN = "unassign";
+
+ static TableState setTableState(Configuration conf, TableName tableName, TableState.State state)
+ throws IOException {
+ try (ClusterConnection conn = (ClusterConnection)ConnectionFactory.createConnection(conf)) {
+ try (Hbck hbck = conn.getHbck()) {
+ return hbck.setTableStateInMeta(new TableState(tableName, state));
+ }
+ }
+ }
+
+ static List<Long> assigns(Configuration conf, List<String> encodedRegionNames)
+ throws IOException {
+ try (ClusterConnection conn = (ClusterConnection)ConnectionFactory.createConnection(conf)) {
+ try (Hbck hbck = conn.getHbck()) {
+ return hbck.assigns(encodedRegionNames);
+ }
+ }
+ }
+
+ static List<Long> unassigns(Configuration conf, List<String> encodedRegionNames)
+ throws IOException {
+ try (ClusterConnection conn = (ClusterConnection)ConnectionFactory.createConnection(conf)) {
+ try (Hbck hbck = conn.getHbck()) {
+ return hbck.unassigns(encodedRegionNames);
+ }
+ }
+ }
private static final String getCommandUsage() {
StringWriter sw = new StringWriter();
PrintWriter writer = new PrintWriter(sw);
+ writer.println();
writer.println("Commands:");
writer.println(" " + SET_TABLE_STATE + " <TABLENAME> <STATE>");
writer.println(" Possible table states: " + Arrays.stream(TableState.State.values()).
@@ -70,6 +103,25 @@ public class HBCK2 {
writer.println(" A value of \\x08\\x00 == ENABLED, \\x08\\x01 == DISABLED, etc.");
writer.println(" An example making table name 'user' ENABLED:");
writer.println(" $ HBCK2 setTableState users ENABLED");
+ writer.println(" Returns whatever the previous table state was.");
+ writer.println();
+ writer.println(" " + ASSIGN + " <ENCODED_REGIONNAME> ...");
+ writer.println(" A 'raw' assign that can be used even during Master initialization.");
+ writer.println(" Skirts Coprocessors. Pass one or more encoded RegionNames:");
+ writer.println(" e.g. 1588230740 is hard-coded encoding for hbase:meta region and");
+ writer.println(" de00010733901a05f5a2a3a382e27dd4 is an example of what a random");
+ writer.println(" user-space encoded Region name looks like. For example:");
+ writer.println(" $ HBCK2 assign 1588230740 de00010733901a05f5a2a3a382e27dd4");
+ writer.println(" Returns the pid of the created AssignProcedure or -1 if none.");
+ writer.println();
+ writer.println(" " + UNASSIGN + " <ENCODED_REGIONNAME> ...");
+ writer.println(" A 'raw' unassign that can be used even during Master initialization.");
+ writer.println(" Skirts Coprocessors. Pass one or more encoded RegionNames:");
+ writer.println(" Skirts Coprocessors. Pass one or more encoded RegionNames:");
+ writer.println(" de00010733901a05f5a2a3a382e27dd4 is an example of what a random");
+ writer.println(" user-space encoded Region name looks like. For example:");
+ writer.println(" $ HBCK2 unassign 1588230740 de00010733901a05f5a2a3a382e27dd4");
+ writer.println(" Returns the pid of the created UnassignProcedure or -1 if none.");
writer.close();
return sw.toString();
}
@@ -84,21 +136,9 @@ public class HBCK2 {
}
HelpFormatter formatter = new HelpFormatter();
formatter.printHelp( "HBCK2 <OPTIONS> COMMAND [<ARGS>]",
- "Options:", options, getCommandUsage());
+ "\nOptions:", options, getCommandUsage());
}
- /**
- * Call the HbckService#setTableState
- * @throws IOException
- */
- static void setTableState(Configuration conf, TableName tableName, TableState.State state)
- throws IOException {
- try (ClusterConnection conn = (ClusterConnection)ConnectionFactory.createConnection(conf)) {
- try (Hbck hbck = conn.getHbck()) {
- hbck.setTableStateInMeta(new TableState(tableName, state));
- }
- }
- }
/**
* @return Return what to exit with.
@@ -162,7 +202,26 @@ public class HBCK2 {
usage(options, command + " takes tablename and state arguments: e.g. user ENABLED");
return EXIT_FAILURE;
}
- setTableState(conf, TableName.valueOf(commands[1]), TableState.State.valueOf(commands[2]));
+ System.out.println(setTableState(conf,
+ TableName.valueOf(commands[1]), TableState.State.valueOf(commands[2])));
+ break;
+
+ case ASSIGN:
+ if (commands.length < 2) {
+ usage(options, command + " takes one or more encoded region names");
+ return EXIT_FAILURE;
+ }
+ System.out.println(pidsToString(
+ assigns(conf, Arrays.stream(commands).skip(1).collect(Collectors.toList()))));
+ break;
+
+ case UNASSIGN:
+ if (commands.length < 2) {
+ usage(options, command + " takes one or more encoded region names");
+ return EXIT_FAILURE;
+ }
+ System.out.println(pidsToString(
+ unassigns(conf, Arrays.stream(commands).skip(1).collect(Collectors.toList()))));
break;
default:
@@ -172,6 +231,10 @@ public class HBCK2 {
return EXIT_SUCCESS;
}
+ private static String pidsToString(List<Long> pids) {
+ return pids.stream().map(i -> i.toString()).collect(Collectors.joining(", "));
+ }
+
public static void main(String [] args) throws IOException {
int exitCode = doWork(args);
if (exitCode != 0) {
diff --git a/hbase-hbck2/src/test/java/org/apache/hbase/TestHBCK2.java b/hbase-hbck2/src/test/java/org/apache/hbase/TestHBCK2.java
index 30afeb2..c875960 100644
--- a/hbase-hbck2/src/test/java/org/apache/hbase/TestHBCK2.java
+++ b/hbase-hbck2/src/test/java/org/apache/hbase/TestHBCK2.java
@@ -17,17 +17,53 @@
*/
package org.apache.hbase;
+import junit.framework.TestCase;
import org.apache.commons.cli.ParseException;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.TableState;
+import org.apache.hadoop.hbase.master.RegionState;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Threads;
+import org.apache.logging.log4j.LogManager;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Rule;
import org.junit.Test;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.PrintStream;
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
public class TestHBCK2 {
+ private static final org.apache.logging.log4j.Logger LOG = LogManager.getLogger(TestHBCK2.class);
+ // private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+ private static final TableName TABLE_NAME = TableName.valueOf(TestHBCK2.class.getSimpleName());
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ /*
+ TEST_UTIL.startMiniCluster(3);
+ TEST_UTIL.createMultiRegionTable(TABLE_NAME, Bytes.toBytes("family1"), 5);
+ */
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ /*
+ TEST_UTIL.shutdownMiniCluster();
+ */
+ }
+
@Test
public void testHelp() throws ParseException, IOException {
// TODO
@@ -42,4 +78,64 @@ public class TestHBCK2 {
String output = os.toString();
assertTrue(output, output.startsWith("usage: HBCK2"));
}
+
+ @Test
+ public void testSetTableStateInMeta() throws IOException {
+ /*
+ TableState state =
+ HBCK2.setTableState(TEST_UTIL.getConfiguration(), TABLE_NAME, TableState.State.DISABLED);
+ TestCase.assertTrue("Found=" + state.getState(), state.isDisabled());
+ // Restore the state.
+ HBCK2.setTableState(TEST_UTIL.getConfiguration(), TABLE_NAME, state.getState());
+ */
+ }
+
+ @Test
+ public void testAssigns() throws IOException {
+ /*
+ try (Admin admin = TEST_UTIL.getConnection().getAdmin()) {
+ List<RegionInfo> regions = admin.getRegions(TABLE_NAME);
+ for (RegionInfo ri: regions) {
+ RegionState rs = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().
+ getRegionStates().getRegionState(ri.getEncodedName());
+ LOG.info("RS: {}", rs.toString());
+ }
+ List<Long> pids = HBCK2.unassigns(TEST_UTIL.getConfiguration(),
+ regions.stream().map(r -> r.getEncodedName()).collect(Collectors.toList()));
+ waitOnPids(pids);
+ for (RegionInfo ri: regions) {
+ RegionState rs = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().
+ getRegionStates().getRegionState(ri.getEncodedName());
+ LOG.info("RS: {}", rs.toString());
+ TestCase.assertTrue(rs.toString(), rs.isClosed());
+ }
+ pids = HBCK2.assigns(TEST_UTIL.getConfiguration(),
+ regions.stream().map(r -> r.getEncodedName()).collect(Collectors.toList()));
+ waitOnPids(pids);
+ for (RegionInfo ri: regions) {
+ RegionState rs = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().
+ getRegionStates().getRegionState(ri.getEncodedName());
+ LOG.info("RS: {}", rs.toString());
+ TestCase.assertTrue(rs.toString(), rs.isOpened());
+ }
+ // What happens if crappy region list passed?
+ pids = HBCK2.assigns(TEST_UTIL.getConfiguration(),
+ Arrays.stream(new String [] {"a", "some rubbish name"}).collect(Collectors.toList()));
+ for (long pid: pids) {
+ assertEquals(org.apache.hadoop.hbase.procedure2.Procedure.NO_PROC_ID, pid);
+ }
+ }
+ */
+ }
+
+ private void waitOnPids(List<Long> pids) {
+ /*
+ for (Long pid: pids) {
+ while (!TEST_UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor().
+ isFinished(pid)) {
+ Threads.sleep(100);
+ }
+ }
+ */
+ }
}
diff --git a/pom.xml b/pom.xml
index 5df605d..b97f120 100644
--- a/pom.xml
+++ b/pom.xml
@@ -123,7 +123,7 @@
<compileSource>1.8</compileSource>
<java.min.version>${compileSource}</java.min.version>
<maven.min.version>3.3.3</maven.min.version>
- <hbase.version>2.1.0</hbase.version>
+ <hbase.version>2.1.1-SNAPSHOT</hbase.version>
<maven.compiler.version>3.6.1</maven.compiler.version>
<surefire.version>2.21.0</surefire.version>
<surefire.provider>surefire-junit47</surefire.provider>