You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by hu...@apache.org on 2020/06/10 23:37:36 UTC
[hbase-operator-tools] branch master updated: HBASE-23927 recommit
the fix with updates for PR review feedback (#64)
This is an automated email from the ASF dual-hosted git repository.
huaxiangsun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase-operator-tools.git
The following commit(s) were added to refs/heads/master by this push:
new edf48e7 HBASE-23927 recommit the fix with updates for PR review feedback (#64)
edf48e7 is described below
commit edf48e71fd3cd6a88d29a9f6cf95ebe83d963085
Author: clarax <cl...@gmail.com>
AuthorDate: Wed Jun 10 16:37:30 2020 -0700
HBASE-23927 recommit the fix with updates for PR review feedback (#64)
HBASE-23927 hbck2 assigns command should accept a file containing a list of region names
Signed-off-by: Nick Dimiduk <nd...@apache.org>
Signed-off-by: Huaxiang Sun <hu...@apache.org>
---
hbase-hbck2/README.md | 9 ++-
.../src/main/java/org/apache/hbase/HBCK2.java | 37 +++++++---
.../src/test/java/org/apache/hbase/TestHBCK2.java | 80 +++++++++++++++-------
3 files changed, 92 insertions(+), 34 deletions(-)
diff --git a/hbase-hbck2/README.md b/hbase-hbck2/README.md
index 39f3711..e7c01ff 100644
--- a/hbase-hbck2/README.md
+++ b/hbase-hbck2/README.md
@@ -124,17 +124,20 @@ Command:
SEE ALSO: reportMissingRegionsInMeta
SEE ALSO: fixMeta
- assigns [OPTIONS] <ENCODED_REGIONNAME>...
+ assigns [OPTIONS] <ENCODED_REGIONNAME/INPUTFILES_FOR_REGIONNAMES>...
Options:
-o,--override override ownership by another procedure
+ -i,--inputFiles take one or more encoded region names
A 'raw' assign that can be used even during Master initialization (if
the -skip flag is specified). Skirts Coprocessors. Pass one or more
encoded region names. 1588230740 is the hard-coded name for the
hbase:meta region and de00010733901a05f5a2a3a382e27dd4 is an example of
what a user-space encoded region name looks like. For example:
- $ HBCK2 assign 1588230740 de00010733901a05f5a2a3a382e27dd4
+ $ HBCK2 assigns 1588230740 de00010733901a05f5a2a3a382e27dd4
Returns the pid(s) of the created AssignProcedure(s) or -1 if none.
-
+ If -i or --inputFiles is specified, pass one or more input file names.
+ Each file contains encoded region names, one per line. For example:
+ $ HBCK2 assigns -i fileName1 fileName2
bypass [OPTIONS] <PID>...
Options:
-o,--override override if procedure is running/stuck
diff --git a/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java b/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java
index 46de2ec..4ab1b4b 100644
--- a/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java
+++ b/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java
@@ -17,10 +17,7 @@
*/
package org.apache.hbase;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.PrintWriter;
-import java.io.StringWriter;
+import java.io.*;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.Arrays;
@@ -35,6 +32,9 @@ import java.util.concurrent.Future;
import java.util.function.Function;
import java.util.stream.Collectors;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.io.LineIterator;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
@@ -109,7 +109,8 @@ public class HBCK2 extends Configured implements org.apache.hadoop.util.Tool {
private static final String ADD_MISSING_REGIONS_IN_META_FOR_TABLES =
"addFsRegionsMissingInMeta";
private static final String REPORT_MISSING_REGIONS_IN_META = "reportMissingRegionsInMeta";
- static final String EXTRA_REGIONS_IN_META = "extraRegionsInMeta";
+ private static final String EXTRA_REGIONS_IN_META = "extraRegionsInMeta";
+
private Configuration conf;
static final String [] MINIMUM_HBCK2_VERSION = {"2.0.3", "2.1.1", "2.2.0", "3.0.0"};
private boolean skipCheck = false;
@@ -280,10 +281,12 @@ public class HBCK2 extends Configured implements org.apache.hadoop.util.Tool {
}
}
- List<Long> assigns(Hbck hbck, String [] args) throws IOException {
+ List<Long> assigns(Hbck hbck, String[] args) throws IOException {
Options options = new Options();
Option override = Option.builder("o").longOpt("override").build();
+ Option inputFile = Option.builder("i").longOpt("inputFiles").build();
options.addOption(override);
+ options.addOption(inputFile);
// Parse command-line.
CommandLineParser parser = new DefaultParser();
CommandLine commandLine;
@@ -294,7 +297,21 @@ public class HBCK2 extends Configured implements org.apache.hadoop.util.Tool {
return null;
}
boolean overrideFlag = commandLine.hasOption(override.getOpt());
- return hbck.assigns(commandLine.getArgList(), overrideFlag);
+
+ List<String> argList = commandLine.getArgList();
+ if (!commandLine.hasOption(inputFile.getOpt())) {
+ return hbck.assigns(argList, overrideFlag);
+ }
+ List<String> assignmentList = new ArrayList<>();
+ for (String filePath : argList) {
+ try (InputStream fileStream = new FileInputStream(filePath)){
+ LineIterator it = IOUtils.lineIterator(fileStream, "UTF-8");
+ while (it.hasNext()) {
+ assignmentList.add(it.nextLine().trim());
+ }
+ }
+ }
+ return hbck.assigns(assignmentList, overrideFlag);
}
List<Long> unassigns(Hbck hbck, String [] args) throws IOException {
@@ -441,9 +458,10 @@ public class HBCK2 extends Configured implements org.apache.hadoop.util.Tool {
}
private static void usageAssigns(PrintWriter writer) {
- writer.println(" " + ASSIGNS + " [OPTIONS] <ENCODED_REGIONNAME>...");
+ writer.println(" " + ASSIGNS + " [OPTIONS] <ENCODED_REGIONNAME/INPUTFILES_FOR_REGIONNAMES>...");
writer.println(" Options:");
writer.println(" -o,--override override ownership by another procedure");
+ writer.println(" -i,--inputFiles take one or more files of encoded region names");
writer.println(" A 'raw' assign that can be used even during Master initialization (if");
writer.println(" the -skip flag is specified). Skirts Coprocessors. Pass one or more");
writer.println(" encoded region names. 1588230740 is the hard-coded name for the");
@@ -451,6 +469,9 @@ public class HBCK2 extends Configured implements org.apache.hadoop.util.Tool {
writer.println(" what a user-space encoded region name looks like. For example:");
writer.println(" $ HBCK2 assigns 1588230740 de00010733901a05f5a2a3a382e27dd4");
writer.println(" Returns the pid(s) of the created AssignProcedure(s) or -1 if none.");
+ writer.println(" If -i or --inputFiles is specified, pass one or more input file names.");
+ writer.println(" Each file contains encoded region names, one per line. For example:");
+ writer.println(" $ HBCK2 assigns -i fileName1 fileName2");
}
private static void usageBypass(PrintWriter writer) {
diff --git a/hbase-hbck2/src/test/java/org/apache/hbase/TestHBCK2.java b/hbase-hbck2/src/test/java/org/apache/hbase/TestHBCK2.java
index 51bc7bc..c120c98 100644
--- a/hbase-hbck2/src/test/java/org/apache/hbase/TestHBCK2.java
+++ b/hbase-hbck2/src/test/java/org/apache/hbase/TestHBCK2.java
@@ -22,12 +22,15 @@ import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.PrintStream;
import java.util.Arrays;
+import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import java.util.Scanner;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.stream.Collectors;
@@ -70,6 +73,8 @@ public class TestHBCK2 {
private static final TableName TABLE_NAME = TableName.valueOf(TestHBCK2.class.getSimpleName());
private static final TableName REGION_STATES_TABLE_NAME = TableName.
valueOf(TestHBCK2.class.getSimpleName() + "-REGIONS_STATES");
+ private final static String ASSIGNS = "assigns";
+ private static final String EXTRA_REGIONS_IN_META = "extraRegionsInMeta";
@Rule
public TestName testName = new TestName();
@@ -127,32 +132,38 @@ public class TestHBCK2 {
getRegionStates().getRegionState(ri.getEncodedName());
LOG.info("RS: {}", rs.toString());
}
- List<String> regionStrs =
- regions.stream().map(RegionInfo::getEncodedName).collect(Collectors.toList());
- String [] regionStrsArray = regionStrs.toArray(new String[] {});
+ String [] regionStrsArray =
+ regions.stream().map(RegionInfo::getEncodedName).toArray(String[]::new);
+
try (ClusterConnection connection = this.hbck2.connect(); Hbck hbck = connection.getHbck()) {
- List<Long> pids = this.hbck2.unassigns(hbck, regionStrsArray);
- waitOnPids(pids);
- for (RegionInfo ri : regions) {
- RegionState rs = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().
- getRegionStates().getRegionState(ri.getEncodedName());
- LOG.info("RS: {}", rs.toString());
- assertTrue(rs.toString(), rs.isClosed());
- }
- pids = this.hbck2.assigns(hbck, regionStrsArray);
+ unassigns(regions, regionStrsArray);
+ List<Long> pids = this.hbck2.assigns(hbck, regionStrsArray);
waitOnPids(pids);
- for (RegionInfo ri : regions) {
- RegionState rs = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().
- getRegionStates().getRegionState(ri.getEncodedName());
- LOG.info("RS: {}", rs.toString());
- assertTrue(rs.toString(), rs.isOpened());
- }
+ validateOpen(regions);
// What happens if crappy region list passed?
pids = this.hbck2.assigns(hbck, Arrays.stream(new String[]{"a", "some rubbish name"}).
- collect(Collectors.toList()).toArray(new String[]{}));
+ collect(Collectors.toList()).toArray(new String[]{}));
for (long pid : pids) {
assertEquals(org.apache.hadoop.hbase.procedure2.Procedure.NO_PROC_ID, pid);
}
+
+ // test input files
+ unassigns(regions, regionStrsArray);
+ String testFile = "inputForAssignsTest";
+ FileOutputStream output = new FileOutputStream(testFile, false);
+ for (String regionStr : regionStrsArray) {
+ output.write((regionStr + System.lineSeparator()).getBytes());
+ }
+ output.close();
+ String result = testRunWithArgs(new String[] {ASSIGNS, "-i", testFile});
+ Scanner scanner = new Scanner(result).useDelimiter("[\\D]+");
+ pids = new ArrayList<>();
+ while (scanner.hasNext()) {
+ pids.add(scanner.nextLong());
+ }
+ scanner.close();
+ waitOnPids(pids);
+ validateOpen(regions);
}
}
}
@@ -266,6 +277,29 @@ public class TestHBCK2 {
assertTrue(result.contains(expectedResult));
}
+ private void unassigns(List<RegionInfo> regions, String[] regionStrsArray) throws IOException {
+ try (ClusterConnection connection = this.hbck2.connect(); Hbck hbck = connection.getHbck()) {
+ List<Long> pids = this.hbck2.unassigns(hbck, regionStrsArray);
+ waitOnPids(pids);
+ }
+ for (RegionInfo ri : regions) {
+ RegionState rs = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().
+ getRegionStates().getRegionState(ri.getEncodedName());
+ LOG.info("RS: {}", rs.toString());
+ assertTrue(rs.toString(), rs.isClosed());
+ }
+ }
+
+
+ private void validateOpen(List<RegionInfo> regions) {
+ for (RegionInfo ri : regions) {
+ RegionState rs = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().
+ getRegionStates().getRegionState(ri.getEncodedName());
+ LOG.info("RS: {}", rs.toString());
+ assertTrue(rs.toString(), rs.isOpened());
+ }
+ }
+
private String testFormatMissingRegionsInMetaReport()
throws IOException {
HBCK2 hbck = new HBCK2(TEST_UTIL.getConfiguration());
@@ -519,18 +553,18 @@ public class TestHBCK2 {
}
private String testFormatExtraRegionsInMetaReport() throws IOException {
- return testFormatExtraRegionsInMeta(new String[]{HBCK2.EXTRA_REGIONS_IN_META });
+ return testRunWithArgs(new String[]{EXTRA_REGIONS_IN_META});
}
private String testFormatExtraRegionsInMetaFix(String table) throws IOException {
if(table!=null) {
- return testFormatExtraRegionsInMeta(new String[] {HBCK2.EXTRA_REGIONS_IN_META, "-f", table});
+ return testRunWithArgs(new String[] {EXTRA_REGIONS_IN_META, "-f", table});
} else {
- return testFormatExtraRegionsInMeta(new String[] {HBCK2.EXTRA_REGIONS_IN_META, "-f"});
+ return testRunWithArgs(new String[] {EXTRA_REGIONS_IN_META, "-f"});
}
}
- private String testFormatExtraRegionsInMeta(String[] args) throws IOException {
+ private String testRunWithArgs(String[] args) throws IOException {
HBCK2 hbck = new HBCK2(TEST_UTIL.getConfiguration());
final StringBuilder builder = new StringBuilder();
PrintStream originalOS = System.out;