You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2016/11/17 07:07:25 UTC
[3/3] hbase git commit: HBASE-16169: Make RegionSizeCalculator
scalable
HBASE-16169: Make RegionSizeCalculator scalable
Signed-off-by: Michael Stack <st...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/80acc2dc
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/80acc2dc
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/80acc2dc
Branch: refs/heads/master
Commit: 80acc2dca5435f46c683f9eae78861bc766589a6
Parents: 48439e5
Author: thiruvel <th...@yahoo-inc.com>
Authored: Wed Nov 9 17:43:38 2016 -0800
Committer: Michael Stack <st...@apache.org>
Committed: Wed Nov 16 23:07:14 2016 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/client/Admin.java | 20 +
.../apache/hadoop/hbase/client/HBaseAdmin.java | 20 +
.../hbase/shaded/protobuf/ProtobufUtil.java | 25 +
.../hbase/shaded/protobuf/RequestConverter.java | 16 +
.../shaded/protobuf/generated/AdminProtos.java | 1678 ++++++++++++++++--
.../src/main/protobuf/Admin.proto | 12 +
.../hbase/regionserver/HRegionServer.java | 2 +-
.../hbase/regionserver/RSRpcServices.java | 32 +
.../hadoop/hbase/util/RegionSizeCalculator.java | 48 +-
.../org/apache/hadoop/hbase/TestRegionLoad.java | 135 ++
.../hadoop/hbase/master/MockRegionServer.java | 9 +
.../hbase/util/TestRegionSizeCalculator.java | 66 +-
12 files changed, 1867 insertions(+), 196 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/80acc2dc/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
index 34c00ce..52b935f 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.NamespaceNotFoundException;
import org.apache.hadoop.hbase.ProcedureInfo;
+import org.apache.hadoop.hbase.RegionLoad;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableExistsException;
import org.apache.hadoop.hbase.TableName;
@@ -1021,6 +1022,25 @@ public interface Admin extends Abortable, Closeable {
ClusterStatus getClusterStatus() throws IOException;
/**
+ * Get {@link RegionLoad} of all regions hosted on a regionserver.
+ *
+ * @param sn region server from which regionload is required.
+ * @return region load map of all regions hosted on a region server
+ * @throws IOException if a remote or network exception occurs
+ */
+ Map<byte[], RegionLoad> getRegionLoad(ServerName sn) throws IOException;
+
+ /**
+ * Get {@link RegionLoad} of all regions hosted on a regionserver for a table.
+ *
+ * @param sn region server from which regionload is required.
+ * @param tableName get region load of regions belonging to the table
+ * @return region load map of all regions of a table hosted on a region server
+ * @throws IOException if a remote or network exception occurs
+ */
+ Map<byte[], RegionLoad> getRegionLoad(ServerName sn, TableName tableName) throws IOException;
+
+ /**
* @return Configuration used by the instance.
*/
Configuration getConfiguration();
http://git-wip-us.apache.org/repos/asf/hbase/blob/80acc2dc/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index 4e195d1..00463f38 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -28,6 +28,7 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.TreeMap;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
@@ -56,6 +57,7 @@ import org.apache.hadoop.hbase.NamespaceNotFoundException;
import org.apache.hadoop.hbase.NotServingRegionException;
import org.apache.hadoop.hbase.ProcedureInfo;
import org.apache.hadoop.hbase.RegionLocations;
+import org.apache.hadoop.hbase.RegionLoad;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableExistsException;
import org.apache.hadoop.hbase.TableName;
@@ -1880,6 +1882,24 @@ public class HBaseAdmin implements Admin {
}
@Override
+ public Map<byte[], RegionLoad> getRegionLoad(final ServerName sn) throws IOException {
+ return getRegionLoad(sn, null);
+ }
+
+ @Override
+ public Map<byte[], RegionLoad> getRegionLoad(final ServerName sn, final TableName tableName)
+ throws IOException {
+ AdminService.BlockingInterface admin = this.connection.getAdmin(sn);
+ HBaseRpcController controller = rpcControllerFactory.newController();
+ List<RegionLoad> regionLoads = ProtobufUtil.getRegionLoad(controller, admin, tableName);
+ Map<byte[], RegionLoad> resultMap = new TreeMap<byte[], RegionLoad>(Bytes.BYTES_COMPARATOR);
+ for (RegionLoad regionLoad : regionLoads) {
+ resultMap.put(regionLoad.getName(), regionLoad);
+ }
+ return resultMap;
+ }
+
+ @Override
public Configuration getConfiguration() {
return this.conf;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/80acc2dc/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
index 2758c26..0c95615 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
@@ -112,6 +112,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetOnlineRe
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetOnlineRegionResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionLoadRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionLoadResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetServerInfoRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetServerInfoResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetStoreFileRequest;
@@ -1677,6 +1679,29 @@ public final class ProtobufUtil {
}
}
+ public static List<org.apache.hadoop.hbase.RegionLoad> getRegionLoad(
+ final RpcController controller, final AdminService.BlockingInterface admin,
+ final TableName tableName) throws IOException {
+ GetRegionLoadRequest request = RequestConverter.buildGetRegionLoadRequest(tableName);
+ GetRegionLoadResponse response;
+ try {
+ response = admin.getRegionLoad(controller, request);
+ } catch (ServiceException se) {
+ throw getRemoteException(se);
+ }
+ return getRegionLoadInfo(response);
+ }
+
+ static List<org.apache.hadoop.hbase.RegionLoad> getRegionLoadInfo(
+ GetRegionLoadResponse regionLoadResponse) {
+ List<org.apache.hadoop.hbase.RegionLoad> regionLoadList =
+ new ArrayList<>(regionLoadResponse.getRegionLoadsCount());
+ for (RegionLoad regionLoad : regionLoadResponse.getRegionLoadsList()) {
+ regionLoadList.add(new org.apache.hadoop.hbase.RegionLoad(regionLoad));
+ }
+ return regionLoadList;
+ }
+
/**
* A helper to close a region given a region name
* using admin protocol.
http://git-wip-us.apache.org/repos/asf/hbase/blob/80acc2dc/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java
index abd1563..61aa5c7 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java
@@ -50,6 +50,8 @@ import org.apache.hadoop.hbase.shaded.com.google.protobuf.UnsafeByteOperations;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CompactRegionRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.FlushRegionRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetOnlineRegionRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionLoadRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionLoadResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetServerInfoRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.OpenRegionRequest;
@@ -750,6 +752,20 @@ public final class RequestConverter {
return builder.build();
}
+ /**
+ * Create a protocol buffer GetRegionLoadRequest for all regions/regions of a table.
+ *
+ * @param tableName the table for which regionLoad should be obtained from RS
+ * @return a protocol buffer GetRegionLoadRequest
+ */
+ public static GetRegionLoadRequest buildGetRegionLoadRequest(final TableName tableName) {
+ GetRegionLoadRequest.Builder builder = GetRegionLoadRequest.newBuilder();
+ if (tableName != null) {
+ builder.setTableName(ProtobufUtil.toProtoTableName(tableName));
+ }
+ return builder.build();
+ }
+
/**
* Create a protocol buffer GetOnlineRegionRequest
*