You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by hu...@apache.org on 2020/06/09 16:27:51 UTC
[hbase-operator-tools] branch master updated: HBCK takes one or
more files for assigns (#62)
This is an automated email from the ASF dual-hosted git repository.
huaxiangsun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase-operator-tools.git
The following commit(s) were added to refs/heads/master by this push:
new aab2503 HBCK takes one or more files for assigns (#62)
aab2503 is described below
commit aab25033ea6657a990b73bbc06ffe949c4a6282d
Author: clarax <cl...@gmail.com>
AuthorDate: Tue Jun 9 09:27:07 2020 -0700
HBCK takes one or more files for assigns (#62)
* HBCK takes one or more files for assigns
* include input file option commandline parsinging in test
* update test for assigns
Co-authored-by: Man Xiong <ma...@MacBook-Pro.local>
---
hbase-hbck2/README.md | 9 ++-
.../src/main/java/org/apache/hbase/HBCK2.java | 41 +++++++++--
.../src/test/java/org/apache/hbase/TestHBCK2.java | 80 +++++++++++++++-------
3 files changed, 100 insertions(+), 30 deletions(-)
diff --git a/hbase-hbck2/README.md b/hbase-hbck2/README.md
index 39f3711..e7c01ff 100644
--- a/hbase-hbck2/README.md
+++ b/hbase-hbck2/README.md
@@ -124,17 +124,20 @@ Command:
SEE ALSO: reportMissingRegionsInMeta
SEE ALSO: fixMeta
- assigns [OPTIONS] <ENCODED_REGIONNAME>...
+ assigns [OPTIONS] <ENCODED_REGIONNAME/INPUTFILES_FOR_REGIONNAMES>...
Options:
-o,--override override ownership by another procedure
+ -i,--inputFiles take one or more encoded region names
A 'raw' assign that can be used even during Master initialization (if
the -skip flag is specified). Skirts Coprocessors. Pass one or more
encoded region names. 1588230740 is the hard-coded name for the
hbase:meta region and de00010733901a05f5a2a3a382e27dd4 is an example of
what a user-space encoded region name looks like. For example:
- $ HBCK2 assign 1588230740 de00010733901a05f5a2a3a382e27dd4
+ $ HBCK2 assigns 1588230740 de00010733901a05f5a2a3a382e27dd4
Returns the pid(s) of the created AssignProcedure(s) or -1 if none.
-
+ If -i or --inputFiles is specified, pass one or more input file names.
+ Each file contains encoded region names, one per line. For example:
+ $ HBCK2 assigns -i fileName1 fileName2
bypass [OPTIONS] <PID>...
Options:
-o,--override override if procedure is running/stuck
diff --git a/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java b/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java
index 46de2ec..f1bd441 100644
--- a/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java
+++ b/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java
@@ -17,6 +17,9 @@
*/
package org.apache.hbase;
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintWriter;
@@ -87,7 +90,6 @@ public class HBCK2 extends Configured implements org.apache.hadoop.util.Tool {
static final int EXIT_FAILURE = 1;
// Commands
private static final String SET_TABLE_STATE = "setTableState";
- private static final String ASSIGNS = "assigns";
private static final String UNASSIGNS = "unassigns";
private static final String BYPASS = "bypass";
private static final String FILESYSTEM = "filesystem";
@@ -109,7 +111,10 @@ public class HBCK2 extends Configured implements org.apache.hadoop.util.Tool {
private static final String ADD_MISSING_REGIONS_IN_META_FOR_TABLES =
"addFsRegionsMissingInMeta";
private static final String REPORT_MISSING_REGIONS_IN_META = "reportMissingRegionsInMeta";
+
static final String EXTRA_REGIONS_IN_META = "extraRegionsInMeta";
+ static final String ASSIGNS = "assigns";
+
private Configuration conf;
static final String [] MINIMUM_HBCK2_VERSION = {"2.0.3", "2.1.1", "2.2.0", "3.0.0"};
private boolean skipCheck = false;
@@ -280,10 +285,12 @@ public class HBCK2 extends Configured implements org.apache.hadoop.util.Tool {
}
}
- List<Long> assigns(Hbck hbck, String [] args) throws IOException {
+ List<Long> assigns(Hbck hbck, String[] args) throws IOException {
Options options = new Options();
Option override = Option.builder("o").longOpt("override").build();
+ Option inputFile = Option.builder("i").longOpt("inputFiles").build();
options.addOption(override);
+ options.addOption(inputFile);
// Parse command-line.
CommandLineParser parser = new DefaultParser();
CommandLine commandLine;
@@ -294,7 +301,29 @@ public class HBCK2 extends Configured implements org.apache.hadoop.util.Tool {
return null;
}
boolean overrideFlag = commandLine.hasOption(override.getOpt());
- return hbck.assigns(commandLine.getArgList(), overrideFlag);
+
+ List<String> argList = commandLine.getArgList();
+ if (!commandLine.hasOption(inputFile.getOpt())) {
+ return hbck.assigns(argList, overrideFlag);
+ } else {
+ List<String> assignmentList = new ArrayList<>();
+ for (String filePath : argList) {
+ try {
+ File file = new File(filePath);
+ FileReader fileReader = new FileReader(file);
+ BufferedReader bufferedReader = new BufferedReader(fileReader);
+ String regionName;
+ while ((regionName = bufferedReader.readLine()) != null) {
+ assignmentList.add(regionName.trim());
+ }
+ fileReader.close();
+ } catch (IOException e) {
+ LOG.error("Error on input file: ", filePath);
+ throw e;
+ }
+ }
+ return hbck.assigns(assignmentList, overrideFlag);
+ }
}
List<Long> unassigns(Hbck hbck, String [] args) throws IOException {
@@ -441,9 +470,10 @@ public class HBCK2 extends Configured implements org.apache.hadoop.util.Tool {
}
private static void usageAssigns(PrintWriter writer) {
- writer.println(" " + ASSIGNS + " [OPTIONS] <ENCODED_REGIONNAME>...");
+ writer.println(" " + ASSIGNS + " [OPTIONS] <ENCODED_REGIONNAME/INPUTFILES_FOR_REGIONNAMES>...");
writer.println(" Options:");
writer.println(" -o,--override override ownership by another procedure");
+ writer.println(" -i,--inputFiles take one or more files of encoded region names");
writer.println(" A 'raw' assign that can be used even during Master initialization (if");
writer.println(" the -skip flag is specified). Skirts Coprocessors. Pass one or more");
writer.println(" encoded region names. 1588230740 is the hard-coded name for the");
@@ -451,6 +481,9 @@ public class HBCK2 extends Configured implements org.apache.hadoop.util.Tool {
writer.println(" what a user-space encoded region name looks like. For example:");
writer.println(" $ HBCK2 assigns 1588230740 de00010733901a05f5a2a3a382e27dd4");
writer.println(" Returns the pid(s) of the created AssignProcedure(s) or -1 if none.");
+ writer.println(" If -i or --inputFiles is specified, pass one or more input file names.");
+ writer.println(" Each file contains encoded region names, one per line. For example:");
+ writer.println(" $ HBCK2 assigns -i fileName1 fileName2");
}
private static void usageBypass(PrintWriter writer) {
diff --git a/hbase-hbck2/src/test/java/org/apache/hbase/TestHBCK2.java b/hbase-hbck2/src/test/java/org/apache/hbase/TestHBCK2.java
index 51bc7bc..5fe084f 100644
--- a/hbase-hbck2/src/test/java/org/apache/hbase/TestHBCK2.java
+++ b/hbase-hbck2/src/test/java/org/apache/hbase/TestHBCK2.java
@@ -22,12 +22,15 @@ import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.PrintStream;
import java.util.Arrays;
+import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import java.util.Scanner;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.stream.Collectors;
@@ -127,32 +130,40 @@ public class TestHBCK2 {
getRegionStates().getRegionState(ri.getEncodedName());
LOG.info("RS: {}", rs.toString());
}
- List<String> regionStrs =
- regions.stream().map(RegionInfo::getEncodedName).collect(Collectors.toList());
- String [] regionStrsArray = regionStrs.toArray(new String[] {});
+ String [] regionStrsArray =
+ regions.stream().map(RegionInfo::getEncodedName).collect(Collectors.toList())
+ .toArray(new String[] {});
+
try (ClusterConnection connection = this.hbck2.connect(); Hbck hbck = connection.getHbck()) {
- List<Long> pids = this.hbck2.unassigns(hbck, regionStrsArray);
- waitOnPids(pids);
- for (RegionInfo ri : regions) {
- RegionState rs = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().
- getRegionStates().getRegionState(ri.getEncodedName());
- LOG.info("RS: {}", rs.toString());
- assertTrue(rs.toString(), rs.isClosed());
- }
- pids = this.hbck2.assigns(hbck, regionStrsArray);
+ unassigns(regions, regionStrsArray);
+ List<Long> pids = this.hbck2.assigns(hbck, regionStrsArray);
waitOnPids(pids);
- for (RegionInfo ri : regions) {
- RegionState rs = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().
- getRegionStates().getRegionState(ri.getEncodedName());
- LOG.info("RS: {}", rs.toString());
- assertTrue(rs.toString(), rs.isOpened());
- }
+ validateOpen(regions);
// What happens if crappy region list passed?
pids = this.hbck2.assigns(hbck, Arrays.stream(new String[]{"a", "some rubbish name"}).
- collect(Collectors.toList()).toArray(new String[]{}));
+ collect(Collectors.toList()).toArray(new String[]{}));
for (long pid : pids) {
assertEquals(org.apache.hadoop.hbase.procedure2.Procedure.NO_PROC_ID, pid);
}
+
+ // test input files
+ unassigns(regions, regionStrsArray);
+ String testFile = "inputForAssignsTest";
+ FileOutputStream output = new FileOutputStream(testFile, false);
+ for (String regionStr : regionStrsArray) {
+ output.write((regionStr + System.lineSeparator()).getBytes());
+ }
+ output.close();
+ String result = testRunWithArgs(new String[] {HBCK2.ASSIGNS, "-i", testFile});
+ Scanner scanner = new Scanner(result).useDelimiter("[\\D]+");
+ pids = new ArrayList<>();
+ while (scanner.hasNext()) {
+ pids.add(scanner.nextLong());
+ }
+ scanner.close();
+ //pids = this.hbck2.assigns(hbck, new String[] {"-i", testFile});
+ waitOnPids(pids);
+ validateOpen(regions);
}
}
}
@@ -266,6 +277,29 @@ public class TestHBCK2 {
assertTrue(result.contains(expectedResult));
}
+ private void unassigns(List<RegionInfo> regions, String[] regionStrsArray) throws IOException {
+ try (ClusterConnection connection = this.hbck2.connect(); Hbck hbck = connection.getHbck()) {
+ List<Long> pids = this.hbck2.unassigns(hbck, regionStrsArray);
+ waitOnPids(pids);
+ }
+ for (RegionInfo ri : regions) {
+ RegionState rs = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().
+ getRegionStates().getRegionState(ri.getEncodedName());
+ LOG.info("RS: {}", rs.toString());
+ assertTrue(rs.toString(), rs.isClosed());
+ }
+ }
+
+
+ private void validateOpen(List<RegionInfo> regions) {
+ for (RegionInfo ri : regions) {
+ RegionState rs = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().
+ getRegionStates().getRegionState(ri.getEncodedName());
+ LOG.info("RS: {}", rs.toString());
+ assertTrue(rs.toString(), rs.isOpened());
+ }
+ }
+
private String testFormatMissingRegionsInMetaReport()
throws IOException {
HBCK2 hbck = new HBCK2(TEST_UTIL.getConfiguration());
@@ -519,18 +553,18 @@ public class TestHBCK2 {
}
private String testFormatExtraRegionsInMetaReport() throws IOException {
- return testFormatExtraRegionsInMeta(new String[]{HBCK2.EXTRA_REGIONS_IN_META });
+ return testRunWithArgs(new String[]{HBCK2.EXTRA_REGIONS_IN_META });
}
private String testFormatExtraRegionsInMetaFix(String table) throws IOException {
if(table!=null) {
- return testFormatExtraRegionsInMeta(new String[] {HBCK2.EXTRA_REGIONS_IN_META, "-f", table});
+ return testRunWithArgs(new String[] {HBCK2.EXTRA_REGIONS_IN_META, "-f", table});
} else {
- return testFormatExtraRegionsInMeta(new String[] {HBCK2.EXTRA_REGIONS_IN_META, "-f"});
+ return testRunWithArgs(new String[] {HBCK2.EXTRA_REGIONS_IN_META, "-f"});
}
}
- private String testFormatExtraRegionsInMeta(String[] args) throws IOException {
+ private String testRunWithArgs(String[] args) throws IOException {
HBCK2 hbck = new HBCK2(TEST_UTIL.getConfiguration());
final StringBuilder builder = new StringBuilder();
PrintStream originalOS = System.out;