You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2018/09/14 05:22:34 UTC
[hbase-operator-tools] branch master updated: Add an HBCK2 tool.
Add command-line processing for the one call that is available in Hbck
Service Interface currently. More to come.
This is an automated email from the ASF dual-hosted git repository.
stack pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase-operator-tools.git
The following commit(s) were added to refs/heads/master by this push:
new 0cf0e0e Add an HBCK2 tool. Add command-line processing for the one call that is available in Hbck Service Interface currently. More to come.
0cf0e0e is described below
commit 0cf0e0ecf2d4a33522e0e273f9310f11aa2eaee6
Author: Michael Stack <st...@apache.org>
AuthorDate: Thu Sep 13 22:21:20 2018 -0700
Add an HBCK2 tool. Add command-line processing for the one
call that is available in Hbck Service Interface currently.
More to come.
---
hbase-hbck2/pom.xml | 129 +++++++++-----------
.../src/main/java/org/apache/hbase/HBCK2.java | 130 +++++++++++++++++++++
hbase-hbck2/src/main/resources/log4j2.xml | 13 +++
.../java/org/apache/hbase/TestHBCK2.java} | 45 ++++---
pom.xml | 76 ++++++++++--
5 files changed, 294 insertions(+), 99 deletions(-)
diff --git a/hbase-hbck2/pom.xml b/hbase-hbck2/pom.xml
index 82e80c9..250b0bc 100644
--- a/hbase-hbck2/pom.xml
+++ b/hbase-hbck2/pom.xml
@@ -32,10 +32,7 @@
<name>Apache HBase - HBCK2</name>
<description>HBCK for HBase 2+</description>
- <dependencies/>
-
<build>
- <sourceDirectory>${project.basedir}/target/java</sourceDirectory>
<resources />
<testResources>
<testResource>
@@ -53,84 +50,72 @@
<artifactId>maven-remote-resources-plugin</artifactId>
</plugin>
<plugin>
- <!--Make it so assembly:single does nothing in here-->
- <artifactId>maven-assembly-plugin</artifactId>
- <configuration>
- <skipAssembly>true</skipAssembly>
- </configuration>
- </plugin>
- <plugin>
<artifactId>maven-surefire-plugin</artifactId>
- <configuration>
- <properties>
- <property />
- </properties>
- </configuration>
</plugin>
<!-- Make a jar and put the sources in the jar -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
- <configuration />
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>3.2.0</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <artifactSet>
+ <excludes>
+ <exclude>classworlds:classworlds</exclude>
+ <exclude>junit:junit</exclude>
+ <exclude>jmock:*</exclude>
+ <exclude>*:xml-apis</exclude>
+ <exclude>org.apache.maven:lib:tests</exclude>
+ <exclude>log4j:log4j:jar:</exclude>
+ </excludes>
+ </artifactSet>
+ </configuration>
+ </execution>
+ </executions>
</plugin>
</plugins>
- <pluginManagement>
- <plugins>
- <!--This plugin's configuration is used to store Eclipse m2e settings
- only. It has no influence on the Maven build itself. -->
- <plugin>
- <groupId>org.eclipse.m2e</groupId>
- <artifactId>lifecycle-mapping</artifactId>
- <version>1.0.0</version>
- <configuration>
- <lifecycleMappingMetadata>
- <pluginExecutions>
- <pluginExecution>
- <pluginExecutionFilter>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-antrun-plugin</artifactId>
- <versionRange>[${maven.antrun.version}]</versionRange>
- <goals>
- <goal>run</goal>
- </goals>
- </pluginExecutionFilter>
- <action>
- <execute/>
- </action>
- </pluginExecution>
- <pluginExecution>
- <pluginExecutionFilter>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-dependency-plugin</artifactId>
- <versionRange>[2.8,)</versionRange>
- <goals>
- <goal>build-classpath</goal>
- </goals>
- </pluginExecutionFilter>
- <action>
- <ignore></ignore>
- </action>
- </pluginExecution>
- <pluginExecution>
- <pluginExecutionFilter>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-compiler-plugin</artifactId>
- <versionRange>[3.2,)</versionRange>
- <goals>
- <goal>compile</goal>
- </goals>
- </pluginExecutionFilter>
- <action>
- <ignore></ignore>
- </action>
- </pluginExecution>
- </pluginExecutions>
- </lifecycleMappingMetadata>
- </configuration>
- </plugin>
- </plugins>
- </pluginManagement>
</build>
+ <dependencies>
+ <!-- https://mvnrepository.com/artifact/commons-cli/commons-cli -->
+ <dependency>
+ <groupId>commons-cli</groupId>
+ <artifactId>commons-cli</artifactId>
+ <version>1.4</version>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.12</version>
+ </dependency>
+ <!-- https://mvnrepository.com/artifact/org.apache.logging.log4j/log4j-core -->
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-api</artifactId>
+ <version>2.11.1</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-core</artifactId>
+ <version>2.11.1</version>
+ </dependency>
+ <!-- https://mvnrepository.com/artifact/org.apache.hbase.thirdparty/hbase-shaded-miscellaneous -->
+ <!-- https://mvnrepository.com/artifact/org.apache.hbase.thirdparty/hbase-shaded-miscellaneous -->
+ <!--Try to use shaded client in hbck2.-->
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-shaded-client</artifactId>
+ <version>2.1.1-SNAPSHOT</version>
+ </dependency>
+ </dependencies>
<profiles>
diff --git a/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java b/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java
new file mode 100644
index 0000000..d88a2de
--- /dev/null
+++ b/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java
@@ -0,0 +1,130 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hbase;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.DefaultParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.ClusterConnection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.Hbck;
+import org.apache.hadoop.hbase.client.TableState;
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.core.config.Configurator;
+
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * HBase fixup tool version 2, for hbase-2.0.0+ clusters.
+ */
+// TODO:
+// + Add bulk assign/unassigns. If 60k OPENING regions, doing it via shell takes 10-60 seconds each.
+public class HBCK2 {
+ private static final Logger logger = LogManager.getLogger(HBCK2.class);
+ private static final String SET_TABLE_STATE = "setTableState";
+
+ private static final String getCommandUsage() {
+ StringWriter sw = new StringWriter();
+ PrintWriter writer = new PrintWriter(sw);
+ writer.println("Commands:");
+ writer.println(" " + SET_TABLE_STATE + " TABLENAME STATE");
+ writer.println("Help:");
+ writer.println(" Possible table states: " + Arrays.stream(TableState.State.values()).
+ map(i -> i.toString()).collect(Collectors.joining(", ")));
+ writer.println("Examples:");
+ writer.println(" $ HBCK2 setTableState users=ENABLED");
+ writer.close();
+ return sw.toString();
+ }
+
+ static void usage(Options options) {
+ usage(options, null);
+ }
+
+ static void usage(Options options, String error) {
+ if (error != null) {
+ System.out.println("ERROR: " + error);
+ }
+ HelpFormatter formatter = new HelpFormatter();
+ formatter.printHelp( "HBCK2 <OPTIONS> COMMAND [<ARGS>]",
+ "Options:", options, getCommandUsage());
+ }
+
+ static void setTableState(TableName tableName, TableState.State state) throws IOException {
+ Configuration conf = HBaseConfiguration.create();
+ try (ClusterConnection conn = (ClusterConnection)ConnectionFactory.createConnection(conf)) {
+ try (Hbck hbck = conn.getHbck()) {
+ hbck.setTableStateInMeta(new TableState(tableName, state));
+ }
+ }
+ }
+
+ public static void main(String [] args) throws ParseException, IOException {
+ // Configure Options.
+ Options options = new Options();
+ Option help = Option.builder("h").longOpt("help").desc("output this help message").build();
+ options.addOption(help);
+ Option debug = Option.builder("d").longOpt("debug").desc("run with debug output").build();
+ options.addOption(debug);
+
+ // Parse command-line.
+ CommandLineParser parser = new DefaultParser();
+ CommandLine commandLine = parser.parse(options, args);
+
+ // Process general options.
+ if (commandLine.hasOption(help.getOpt()) || commandLine.getArgList().isEmpty()) {
+ usage(options);
+ System.exit(0);
+ }
+ if (commandLine.hasOption(debug.getOpt())) {
+ Configurator.setRootLevel(Level.DEBUG);
+ }
+
+ // Now process commands.
+ String [] commands = commandLine.getArgs();
+ String command = commands[0];
+ switch (command) {
+ case SET_TABLE_STATE:
+ if (commands.length < 3) {
+ usage(options, command + " takes a table name = state argument, e.g. user ENABLED");
+ System.exit(1);
+ }
+ setTableState(TableName.valueOf(commands[1]), TableState.State.valueOf(commands[2]));
+ break;
+
+ default:
+ usage(options, "Unsupported command: " + command);
+ System.exit(1);
+ }
+ }
+}
\ No newline at end of file
diff --git a/hbase-hbck2/src/main/resources/log4j2.xml b/hbase-hbck2/src/main/resources/log4j2.xml
new file mode 100644
index 0000000..9a62a18
--- /dev/null
+++ b/hbase-hbck2/src/main/resources/log4j2.xml
@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Configuration status="INFO">
+ <Appenders>
+ <Console name="Console" target="SYSTEM_OUT">
+ <PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/>
+ </Console>
+ </Appenders>
+ <Loggers>
+ <Root level="error">
+ <AppenderRef ref="Console"/>
+ </Root>
+ </Loggers>
+</Configuration>
diff --git a/hbase-hbck2/src/main/avro/HbaseKafkaEvent.avro b/hbase-hbck2/src/test/java/org/apache/hbase/TestHBCK2.java
similarity index 53%
rename from hbase-hbck2/src/main/avro/HbaseKafkaEvent.avro
rename to hbase-hbck2/src/test/java/org/apache/hbase/TestHBCK2.java
index ec88627..e1c0d2b 100644
--- a/hbase-hbck2/src/main/avro/HbaseKafkaEvent.avro
+++ b/hbase-hbck2/src/test/java/org/apache/hbase/TestHBCK2.java
@@ -1,4 +1,4 @@
-/*
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -6,25 +6,36 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
-*/
-{"namespace": "org.apache.hadoop.hbase.kafka",
- "type": "record",
- "name": "HBaseKafkaEvent",
- "fields": [
- {"name": "key", "type": "bytes"},
- {"name": "timestamp", "type": "long" },
- {"name": "delete", "type": "boolean" },
- {"name": "value", "type": "bytes"},
- {"name": "qualifier", "type": "bytes"},
- {"name": "family", "type": "bytes"},
- {"name": "table", "type": "bytes"}
- ]
+ */
+package org.apache.hbase;
+
+import org.apache.commons.cli.ParseException;
+import org.junit.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintStream;
+
+public class TestHBCK2 {
+ @Test
+ public void testHelp() throws ParseException, IOException {
+ // TODO
+ OutputStream os = new ByteArrayOutputStream();
+ PrintStream stream = new PrintStream(os);
+ PrintStream oldOut = System.out;
+ System.setOut(stream);
+ stream.close();
+ os.close();
+ System.setOut(oldOut);
+ System.out.println(os.toString());
+ }
}
diff --git a/pom.xml b/pom.xml
index a2b69f8..7c8ce5a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -125,6 +125,8 @@
<maven.min.version>3.3.3</maven.min.version>
<hbase.version>2.1.0</hbase.version>
<maven.compiler.version>3.6.1</maven.compiler.version>
+ <surefire.version>2.21.0</surefire.version>
+ <surefire.provider>surefire-junit47</surefire.provider>
</properties>
<dependencyManagement>
<dependencies>
@@ -169,6 +171,20 @@
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <version>${maven.dependency.version}</version>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <version>${maven.javadoc.version}</version>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-remote-resources-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-release-plugin</artifactId>
<configuration>
<!--You need this profile. It'll sign your artifacts.
@@ -203,6 +219,56 @@
<version>1.9.1</version>
</plugin>
<plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>${surefire.version}</version>
+ <dependencies>
+ <!-- by default surefire selects dynamically the connector to the unit tests
+ tool. We want to use always the same as the different connectors can have different
+ bugs and behaviour. -->
+ <dependency>
+ <groupId>org.apache.maven.surefire</groupId>
+ <artifactId>${surefire.provider}</artifactId>
+ <version>${surefire.version}</version>
+ </dependency>
+ </dependencies>
+ <!-- Generic testing configuration for all packages -->
+ <configuration>
+ <failIfNoTests>false</failIfNoTests>
+ <reuseForks>false</reuseForks>
+ <testFailureIgnore>${surefire.testFailureIgnore}</testFailureIgnore>
+ <forkedProcessTimeoutInSeconds>${surefire.timeout}</forkedProcessTimeoutInSeconds>
+ <redirectTestOutputToFile>${test.output.tofile}</redirectTestOutputToFile>
+ <systemPropertyVariables>
+ <test.build.classes>${test.build.classes}</test.build.classes>
+ </systemPropertyVariables>
+ <excludes>
+ <!-- users can add -D option to skip particular test classes
+ ex: mvn test -Dtest.exclude.pattern=**/TestFoo.java,**/TestBar.java
+ -->
+ <exclude>${test.exclude.pattern}</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-report-plugin</artifactId>
+ <version>${surefire.version}</version>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-clean-plugin</artifactId>
+ <configuration>
+ <filesets>
+ <fileset>
+ <!--dfs tests have build dir hardcoded. Clean it as part of
+ clean target-->
+ <directory>build</directory>
+ </fileset>
+ </filesets>
+ </configuration>
+ </plugin>
+ <plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
<version>0.11</version>
@@ -261,11 +327,6 @@
<timestampPropertyName>build.year</timestampPropertyName>
</configuration>
</plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <version>3.0.0</version>
- </plugin>
</plugins>
</pluginManagement>
<plugins>
@@ -334,11 +395,6 @@
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<version>3.0.0</version>
- <configuration>
- <descriptors>
- <descriptor>src/main/assembly/src.xml</descriptor>
- </descriptors>
- </configuration>
</plugin>
</plugins>
</build>