You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2017/10/24 01:03:05 UTC
[14/14] hbase git commit: HBASE-15631 Backport Regionserver Groups
(HBASE-6721) to branch-1 (Francis Liu and Andrew Purtell)
HBASE-15631 Backport Regionserver Groups (HBASE-6721) to branch-1 (Francis Liu and Andrew Purtell)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/64328cae
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/64328cae
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/64328cae
Branch: refs/heads/branch-1
Commit: 64328caef0bb712bb69d0241b4b8b3474a82702c
Parents: 795f48c
Author: Andrew Purtell <ap...@apache.org>
Authored: Mon Oct 23 14:15:06 2017 -0700
Committer: Andrew Purtell <ap...@apache.org>
Committed: Mon Oct 23 17:10:33 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/ServerName.java | 25 +-
.../org/apache/hadoop/hbase/net/Address.java | 89 +
hbase-it/pom.xml | 41 +
.../hbase/rsgroup/IntegrationTestRSGroup.java | 99 +
hbase-protocol/pom.xml | 2 +
.../hbase/protobuf/generated/ClientProtos.java | 16 +-
.../protobuf/generated/RSGroupAdminProtos.java | 13571 +++++++++++++++++
.../hbase/protobuf/generated/RSGroupProtos.java | 1332 ++
hbase-protocol/src/main/protobuf/RSGroup.proto | 35 +
.../src/main/protobuf/RSGroupAdmin.proto | 149 +
hbase-rsgroup/pom.xml | 278 +
.../hadoop/hbase/rsgroup/RSGroupAdmin.java | 92 +
.../hbase/rsgroup/RSGroupAdminClient.java | 212 +
.../hbase/rsgroup/RSGroupAdminEndpoint.java | 1049 ++
.../hbase/rsgroup/RSGroupAdminServer.java | 516 +
.../hbase/rsgroup/RSGroupBasedLoadBalancer.java | 431 +
.../hadoop/hbase/rsgroup/RSGroupInfo.java | 190 +
.../hbase/rsgroup/RSGroupInfoManager.java | 116 +
.../hbase/rsgroup/RSGroupInfoManagerImpl.java | 795 +
.../hbase/rsgroup/RSGroupProtobufUtil.java | 61 +
.../hadoop/hbase/rsgroup/RSGroupSerDe.java | 88 +
.../hbase/rsgroup/RSGroupableBalancer.java | 32 +
.../balancer/TestRSGroupBasedLoadBalancer.java | 573 +
.../hadoop/hbase/rsgroup/TestRSGroups.java | 300 +
.../hadoop/hbase/rsgroup/TestRSGroupsBase.java | 815 +
.../hbase/rsgroup/TestRSGroupsOfflineMode.java | 187 +
.../rsgroup/VerifyingRSGroupAdminClient.java | 155 +
.../hbase/tmpl/master/MasterStatusTmpl.jamon | 2 +
.../apache/hadoop/hbase/LocalHBaseCluster.java | 3 +
.../BaseMasterAndRegionObserver.java | 62 +
.../hbase/coprocessor/BaseMasterObserver.java | 63 +
.../hbase/coprocessor/MasterObserver.java | 113 +
.../hadoop/hbase/master/AssignmentManager.java | 16 +-
.../org/apache/hadoop/hbase/master/HMaster.java | 5 +
.../hadoop/hbase/master/LoadBalancer.java | 3 +
.../hbase/master/MasterCoprocessorHost.java | 160 +
.../hadoop/hbase/master/MasterServices.java | 5 +
.../hbase/security/access/AccessController.java | 37 +
.../hbase/coprocessor/TestMasterObserver.java | 61 +
.../hbase/master/MockNoopMasterServices.java | 5 +
.../master/TestAssignmentManagerOnCluster.java | 127 +-
.../hadoop/hbase/master/TestCatalogJanitor.java | 3 +
.../hbase/master/TestMasterStatusServlet.java | 12 +-
.../normalizer/TestSimpleRegionNormalizer.java | 2 +-
.../security/access/TestAccessController.java | 75 +
hbase-shell/pom.xml | 35 +
hbase-shell/src/main/ruby/hbase.rb | 1 +
hbase-shell/src/main/ruby/hbase/hbase.rb | 4 +
.../src/main/ruby/hbase/rsgroup_admin.rb | 164 +
hbase-shell/src/main/ruby/shell.rb | 22 +
hbase-shell/src/main/ruby/shell/commands.rb | 4 +
.../src/main/ruby/shell/commands/add_rsgroup.rb | 39 +
.../main/ruby/shell/commands/balance_rsgroup.rb | 37 +
.../src/main/ruby/shell/commands/get_rsgroup.rb | 43 +
.../ruby/shell/commands/get_server_rsgroup.rb | 39 +
.../ruby/shell/commands/get_table_rsgroup.rb | 40 +
.../main/ruby/shell/commands/list_rsgroups.rb | 49 +
.../ruby/shell/commands/move_servers_rsgroup.rb | 37 +
.../commands/move_servers_tables_rsgroup.rb | 37 +
.../ruby/shell/commands/move_tables_rsgroup.rb | 37 +
.../main/ruby/shell/commands/remove_rsgroup.rb | 37 +
.../apache/hadoop/hbase/client/TestShell.java | 2 +-
.../hbase/client/rsgroup/TestShellRSGroups.java | 111 +
.../src/test/ruby/shell/rsgroup_shell_test.rb | 96 +
hbase-shell/src/test/ruby/test_helper.rb | 4 +
pom.xml | 23 +
66 files changed, 22843 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/64328cae/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerName.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerName.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerName.java
index c90e7e1..339f588 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerName.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerName.java
@@ -24,6 +24,7 @@ import com.google.protobuf.InvalidProtocolBufferException;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.classification.InterfaceStability;
import org.apache.hadoop.hbase.exceptions.DeserializationException;
+import org.apache.hadoop.hbase.net.Address;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
import org.apache.hadoop.hbase.util.Addressing;
@@ -106,7 +107,7 @@ public class ServerName implements Comparable<ServerName>, Serializable {
this.hostnameOnly = hostname;
this.port = port;
this.startcode = startcode;
- this.servername = getServerName(this.hostnameOnly, port, startcode);
+ this.servername = getServerName(hostname, port, startcode);
}
/**
@@ -402,4 +403,26 @@ public class ServerName implements Comparable<ServerName>, Serializable {
int port = Addressing.parsePort(str);
return valueOf(hostname, port, -1L);
}
+
+ /**
+ * @return an Address constructed from the hostname and port carried by this ServerName
+ */
+ public Address getAddress() {
+ return Address.fromParts(getHostname(), getPort());
+ }
+
+ /**
+ * @param left
+ * @param right
+ * @return True if <code>other</code> has same hostname and port.
+ */
+ public static boolean isSameAddress(final ServerName left,
+ final ServerName right) {
+ // TODO: Make this left.getAddress().equals(right.getAddress())
+ if (left == null) return false;
+ if (right == null) return false;
+ return left.getHostname().compareToIgnoreCase(right.getHostname()) == 0 &&
+ left.getPort() == right.getPort();
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/64328cae/hbase-common/src/main/java/org/apache/hadoop/hbase/net/Address.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/net/Address.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/net/Address.java
new file mode 100644
index 0000000..15b4960
--- /dev/null
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/net/Address.java
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.net;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+
+import com.google.common.net.HostAndPort;
+
+/**
+ * An immutable type to hold a hostname and port combo, like an Endpoint
+ * or java.net.InetSocketAddress (but without danger of our calling
+ * resolve -- we do NOT want a resolve happening every time we want
+ * to hold a hostname and port combo). This class is also <<Comparable>>.
+ * <p>In implementation this class is a facade over Guava's {@link HostAndPort}.
+ * We cannot have Guava classes in our API hence this Type.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Stable
+public class Address implements Comparable<Address> {
+ private HostAndPort hostAndPort;
+
+ private Address(HostAndPort hostAndPort) {
+ this.hostAndPort = hostAndPort;
+ }
+
+ public static Address fromParts(String hostname, int port) {
+ return new Address(HostAndPort.fromParts(hostname, port));
+ }
+
+ public static Address fromString(String hostnameAndPort) {
+ return new Address(HostAndPort.fromString(hostnameAndPort));
+ }
+
+ public String getHostname() {
+ return this.hostAndPort.getHostText();
+ }
+
+ public int getPort() {
+ return this.hostAndPort.getPort();
+ }
+
+ @Override
+ public String toString() {
+ return this.hostAndPort.toString();
+ }
+
+ @Override
+ // Don't use HostAndPort equals... It is wonky including
+ // ipv6 brackets
+ public boolean equals(Object other) {
+ if (this == other) {
+ return true;
+ }
+ if (other instanceof Address) {
+ Address that = (Address)other;
+ return this.getHostname().equals(that.getHostname()) &&
+ this.getPort() == that.getPort();
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return this.getHostname().hashCode() ^ getPort();
+ }
+
+ @Override
+ public int compareTo(Address that) {
+ int compare = this.getHostname().compareTo(that.getHostname());
+ if (compare != 0) return compare;
+ return this.getPort() - that.getPort();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/64328cae/hbase-it/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-it/pom.xml b/hbase-it/pom.xml
index 715bcac..8839a9c 100644
--- a/hbase-it/pom.xml
+++ b/hbase-it/pom.xml
@@ -265,6 +265,47 @@
</dependencies>
<profiles>
+ <profile>
+ <id>rsgroup</id>
+ <activation>
+ <property>
+ <name>!skip-rsgroup</name>
+ </property>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-rsgroup</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-rsgroup</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>add-test-source</id>
+ <goals>
+ <goal>add-test-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>src/test/rsgroup</source>
+ </sources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
<!-- Skip the tests in this module -->
<profile>
<id>skipIntegrationTests</id>
http://git-wip-us.apache.org/repos/asf/hbase/blob/64328cae/hbase-it/src/test/rsgroup/org/apache/hadoop/hbase/rsgroup/IntegrationTestRSGroup.java
----------------------------------------------------------------------
diff --git a/hbase-it/src/test/rsgroup/org/apache/hadoop/hbase/rsgroup/IntegrationTestRSGroup.java b/hbase-it/src/test/rsgroup/org/apache/hadoop/hbase/rsgroup/IntegrationTestRSGroup.java
new file mode 100644
index 0000000..e5bb995
--- /dev/null
+++ b/hbase-it/src/test/rsgroup/org/apache/hadoop/hbase/rsgroup/IntegrationTestRSGroup.java
@@ -0,0 +1,99 @@
+/**
+ * Copyright The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.rsgroup;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.IntegrationTestingUtility;
+import org.apache.hadoop.hbase.Waiter;
+import org.apache.hadoop.hbase.testclassification.IntegrationTests;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Runs all of the units tests defined in TestGroupBase
+ * as an integration test.
+ * Requires TestRSGroupBase.NUM_SLAVE_BASE servers to run.
+ */
+@Category(IntegrationTests.class)
+public class IntegrationTestRSGroup extends TestRSGroupsBase {
+ //Integration specific
+ private final static Log LOG = LogFactory.getLog(IntegrationTestRSGroup.class);
+ private static boolean initialized = false;
+
+ @Before
+ public void beforeMethod() throws Exception {
+ if(!initialized) {
+ LOG.info("Setting up IntegrationTestGroup");
+ LOG.info("Initializing cluster with " + NUM_SLAVES_BASE + " servers");
+ TEST_UTIL = new IntegrationTestingUtility();
+ ((IntegrationTestingUtility)TEST_UTIL).initializeCluster(NUM_SLAVES_BASE);
+ //set shared configs
+ admin = TEST_UTIL.getHBaseAdmin();
+ cluster = TEST_UTIL.getHBaseClusterInterface();
+ rsGroupAdmin = new VerifyingRSGroupAdminClient(new RSGroupAdminClient(TEST_UTIL.getConnection()),
+ TEST_UTIL.getConfiguration());
+ LOG.info("Done initializing cluster");
+ initialized = true;
+ //cluster may not be clean
+ //cleanup when initializing
+ afterMethod();
+ }
+ }
+
+ @After
+ public void afterMethod() throws Exception {
+ LOG.info("Cleaning up previous test run");
+ //cleanup previous artifacts
+ deleteTableIfNecessary();
+ deleteNamespaceIfNecessary();
+ deleteGroups();
+ admin.setBalancerRunning(true, true);
+
+ LOG.info("Restoring the cluster");
+ ((IntegrationTestingUtility)TEST_UTIL).restoreCluster();
+ LOG.info("Done restoring the cluster");
+
+ TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
+ @Override
+ public boolean evaluate() throws Exception {
+ LOG.info("Waiting for cleanup to finish "+ rsGroupAdmin.listRSGroups());
+ //Might be greater since moving servers back to default
+ //is after starting a server
+ return rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().size()
+ >= NUM_SLAVES_BASE;
+ }
+ });
+
+ TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
+ @Override
+ public boolean evaluate() throws Exception {
+ LOG.info("Waiting for regionservers to be registered "+ rsGroupAdmin.listRSGroups());
+ //Might be greater since moving servers back to default
+ //is after starting a server
+ return rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().size()
+ == getNumServers();
+ }
+ });
+
+ LOG.info("Done cleaning up previous test run");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/64328cae/hbase-protocol/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-protocol/pom.xml b/hbase-protocol/pom.xml
index 8588a8c..a3b9be1 100644
--- a/hbase-protocol/pom.xml
+++ b/hbase-protocol/pom.xml
@@ -195,6 +195,8 @@
<include>RegionNormalizer.proto</include>
<include>RegionServerStatus.proto</include>
<include>RowProcessor.proto</include>
+ <include>RSGroup.proto</include>
+ <include>RSGroupAdmin.proto</include>
<include>SecureBulkLoad.proto</include>
<include>Snapshot.proto</include>
<include>Table.proto</include>
http://git-wip-us.apache.org/repos/asf/hbase/blob/64328cae/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ClientProtos.java
----------------------------------------------------------------------
diff --git a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ClientProtos.java b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ClientProtos.java
index 06ebc65..35fddc2 100644
--- a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ClientProtos.java
+++ b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ClientProtos.java
@@ -1986,7 +1986,7 @@ public final class ClientProtos {
* <code>optional bool load_column_families_on_demand = 14;</code>
*
* <pre>
- * DO NOT add defaults to load_column_families_on_demand.
+ * DO NOT add defaults to load_column_families_on_demand.
* </pre>
*/
boolean hasLoadColumnFamiliesOnDemand();
@@ -1994,7 +1994,7 @@ public final class ClientProtos {
* <code>optional bool load_column_families_on_demand = 14;</code>
*
* <pre>
- * DO NOT add defaults to load_column_families_on_demand.
+ * DO NOT add defaults to load_column_families_on_demand.
* </pre>
*/
boolean getLoadColumnFamiliesOnDemand();
@@ -2515,7 +2515,7 @@ public final class ClientProtos {
* <code>optional bool load_column_families_on_demand = 14;</code>
*
* <pre>
- * DO NOT add defaults to load_column_families_on_demand.
+ * DO NOT add defaults to load_column_families_on_demand.
* </pre>
*/
public boolean hasLoadColumnFamiliesOnDemand() {
@@ -2525,7 +2525,7 @@ public final class ClientProtos {
* <code>optional bool load_column_families_on_demand = 14;</code>
*
* <pre>
- * DO NOT add defaults to load_column_families_on_demand.
+ * DO NOT add defaults to load_column_families_on_demand.
* </pre>
*/
public boolean getLoadColumnFamiliesOnDemand() {
@@ -4577,7 +4577,7 @@ public final class ClientProtos {
* <code>optional bool load_column_families_on_demand = 14;</code>
*
* <pre>
- * DO NOT add defaults to load_column_families_on_demand.
+ * DO NOT add defaults to load_column_families_on_demand.
* </pre>
*/
public boolean hasLoadColumnFamiliesOnDemand() {
@@ -4587,7 +4587,7 @@ public final class ClientProtos {
* <code>optional bool load_column_families_on_demand = 14;</code>
*
* <pre>
- * DO NOT add defaults to load_column_families_on_demand.
+ * DO NOT add defaults to load_column_families_on_demand.
* </pre>
*/
public boolean getLoadColumnFamiliesOnDemand() {
@@ -4597,7 +4597,7 @@ public final class ClientProtos {
* <code>optional bool load_column_families_on_demand = 14;</code>
*
* <pre>
- * DO NOT add defaults to load_column_families_on_demand.
+ * DO NOT add defaults to load_column_families_on_demand.
* </pre>
*/
public Builder setLoadColumnFamiliesOnDemand(boolean value) {
@@ -4610,7 +4610,7 @@ public final class ClientProtos {
* <code>optional bool load_column_families_on_demand = 14;</code>
*
* <pre>
- * DO NOT add defaults to load_column_families_on_demand.
+ * DO NOT add defaults to load_column_families_on_demand.
* </pre>
*/
public Builder clearLoadColumnFamiliesOnDemand() {