You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2019/12/11 16:56:34 UTC
[hbase] branch branch-2.2 updated: HBASE-23554 Encoded regionname
to regionname utility (#923)
This is an automated email from the ASF dual-hosted git repository.
stack pushed a commit to branch branch-2.2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.2 by this push:
new 3223c50 HBASE-23554 Encoded regionname to regionname utility (#923)
3223c50 is described below
commit 3223c50de7af77726739066fa4b471a26df528ca
Author: Michael Stack <sa...@users.noreply.github.com>
AuthorDate: Wed Dec 11 08:56:23 2019 -0800
HBASE-23554 Encoded regionname to regionname utility (#923)
Adds shell command regioninfo:
hbase(main):001:0> regioninfo '0e6aa5c19ae2b2627649dc7708ce27d0'
{ENCODED => 0e6aa5c19ae2b2627649dc7708ce27d0, NAME => 'TestTable,,1575941375972.0e6aa5c19ae2b2627649dc7708ce27d0.', STARTKEY => '', ENDKEY => '00000000000000000000299441'}
Took 0.4737 seconds
Signed-off-by: Sean Busbey <bu...@apache.org>
Signed-off-by: Duo Zhang <zh...@apache.org>
---
.../java/org/apache/hadoop/hbase/HRegionInfo.java | 3 +-
.../org/apache/hadoop/hbase/client/RegionInfo.java | 61 ++++++++++++----------
.../hadoop/hbase/shaded/protobuf/ProtobufUtil.java | 18 +++----
.../hadoop/hbase/master/MasterRpcServices.java | 47 ++++++++++++-----
.../hadoop/hbase/regionserver/RSRpcServices.java | 3 ++
.../org/apache/hadoop/hbase/client/TestAdmin2.java | 28 ++++++++++
hbase-shell/src/main/ruby/shell.rb | 1 +
.../src/main/ruby/shell/commands/regioninfo.rb | 47 +++++++++++++++++
8 files changed, 156 insertions(+), 52 deletions(-)
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
index fc03926..77602d5 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
@@ -456,8 +456,7 @@ public class HRegionInfo implements RegionInfo, Comparable<HRegionInfo> {
*/
@Deprecated
@InterfaceAudience.Private
- public static byte [][] parseRegionName(final byte [] regionName)
- throws IOException {
+ public static byte [][] parseRegionName(final byte [] regionName) throws IOException {
return RegionInfo.parseRegionName(regionName);
}
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionInfo.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionInfo.java
index 7a03e05..2f9e88d 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionInfo.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionInfo.java
@@ -37,7 +37,6 @@ import org.apache.hadoop.hbase.util.HashKey;
import org.apache.hadoop.hbase.util.JenkinsHash;
import org.apache.hadoop.hbase.util.MD5Hash;
import org.apache.hadoop.io.DataInputBuffer;
-import org.apache.hadoop.util.StringUtils;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
@@ -346,18 +345,15 @@ public interface RegionInfo {
return parseRegionName(regionName)[1];
}
- @InterfaceAudience.Private
- static boolean isEncodedRegionName(byte[] regionName) throws IOException {
- try {
- parseRegionName(regionName);
- return false;
- } catch (IOException e) {
- if (StringUtils.stringifyException(e)
- .contains(INVALID_REGION_NAME_FORMAT_MESSAGE)) {
- return true;
- }
- throw e;
- }
+ /**
+ * Figure if the passed bytes represent an encoded region name or not.
+ * @param regionName A Region name either encoded or not.
+ * @return True if <code>regionName</code> represents an encoded name.
+ */
+ @InterfaceAudience.Private // For use by internals only.
+ public static boolean isEncodedRegionName(byte[] regionName) throws IOException {
+ // If not parseable as region name, presume encoded. TODO: add stringency; e.g. if hex.
+ return parseRegionNameOrReturnNull(regionName) == null && regionName.length <= MD5_HEX_LENGTH;
}
/**
@@ -596,15 +592,28 @@ public interface RegionInfo {
/**
* Separate elements of a regionName.
- * @return Array of byte[] containing tableName, startKey and id
+ * @return Array of byte[] containing tableName, startKey and id OR null if
+ * not parseable as a region name.
+ * @throws IOException if not parseable as regionName.
*/
- static byte [][] parseRegionName(final byte[] regionName)
- throws IOException {
- // Region name is of the format:
- // tablename,startkey,regionIdTimestamp[_replicaId][.encodedName.]
- // startkey can contain the delimiter (',') so we parse from the start and end
+ static byte [][] parseRegionName(final byte[] regionName) throws IOException {
+ byte [][] result = parseRegionNameOrReturnNull(regionName);
+ if (result == null) {
+ throw new IOException(INVALID_REGION_NAME_FORMAT_MESSAGE + ": " + Bytes.toStringBinary(regionName));
+ }
+ return result;
+ }
- // parse from start
+ /**
+ * Separate elements of a regionName.
+ * Region name is of the format:
+ * <code>tablename,startkey,regionIdTimestamp[_replicaId][.encodedName.]</code>.
+ * Startkey can contain the delimiter (',') so we parse from the start and then parse from
+ * the end.
+ * @return Array of byte[] containing tableName, startKey and id OR null if not parseable
+ * as a region name.
+ */
+ static byte [][] parseRegionNameOrReturnNull(final byte[] regionName) {
int offset = -1;
for (int i = 0; i < regionName.length; i++) {
if (regionName[i] == HConstants.DELIMITER) {
@@ -613,8 +622,7 @@ public interface RegionInfo {
}
}
if (offset == -1) {
- throw new IOException(INVALID_REGION_NAME_FORMAT_MESSAGE
- + ": " + Bytes.toStringBinary(regionName));
+ return null;
}
byte[] tableName = new byte[offset];
System.arraycopy(regionName, 0, tableName, 0, offset);
@@ -622,9 +630,9 @@ public interface RegionInfo {
int endOffset = regionName.length;
// check whether regionName contains encodedName
- if (regionName.length > MD5_HEX_LENGTH + 2
- && regionName[regionName.length-1] == ENC_SEPARATOR
- && regionName[regionName.length-MD5_HEX_LENGTH-2] == ENC_SEPARATOR) {
+ if (regionName.length > MD5_HEX_LENGTH + 2 &&
+ regionName[regionName.length-1] == ENC_SEPARATOR &&
+ regionName[regionName.length-MD5_HEX_LENGTH-2] == ENC_SEPARATOR) {
endOffset = endOffset - MD5_HEX_LENGTH - 2;
}
@@ -645,8 +653,7 @@ public interface RegionInfo {
}
}
if (offset == -1) {
- throw new IOException(INVALID_REGION_NAME_FORMAT_MESSAGE
- + ": " + Bytes.toStringBinary(regionName));
+ return null;
}
byte [] startKey = HConstants.EMPTY_BYTE_ARRAY;
if(offset != tableName.length + 1) {
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
index 354e41c..5c6b38a 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
@@ -1714,21 +1714,21 @@ public final class ProtobufUtil {
// Start helpers for Admin
/**
- * A helper to retrieve region info given a region name
- * using admin protocol.
+ * A helper to retrieve region info given a region name or an
+ * encoded region name using admin protocol.
*
- * @param admin
- * @param regionName
* @return the retrieved region info
- * @throws IOException
*/
- public static org.apache.hadoop.hbase.client.RegionInfo getRegionInfo(final RpcController controller,
- final AdminService.BlockingInterface admin, final byte[] regionName) throws IOException {
+ public static org.apache.hadoop.hbase.client.RegionInfo getRegionInfo(
+ final RpcController controller, final AdminService.BlockingInterface admin,
+ final byte[] regionName) throws IOException {
try {
GetRegionInfoRequest request =
+ org.apache.hadoop.hbase.client.RegionInfo.isEncodedRegionName(regionName)?
+ GetRegionInfoRequest.newBuilder().setRegion(RequestConverter.
+ buildRegionSpecifier(RegionSpecifierType.ENCODED_REGION_NAME, regionName)).build():
RequestConverter.buildGetRegionInfoRequest(regionName);
- GetRegionInfoResponse response =
- admin.getRegionInfo(controller, request);
+ GetRegionInfoResponse response = admin.getRegionInfo(controller, request);
return toRegionInfo(response.getRegionInfo());
} catch (ServiceException se) {
throw getRemoteException(se);
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
index d7661a3..c7598ec 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
@@ -1,5 +1,4 @@
-/**
- *
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -1696,24 +1695,44 @@ public class MasterRpcServices extends RSRpcServices
}
}
+ /**
+ * This method implements Admin getRegionInfo. On RegionServer, it is
+ * able to return RegionInfo and detail. On Master, it just returns
+ * RegionInfo. On Master it has been hijacked to return Mob detail.
+ * Master implementation is good for querying full region name if
+ * you only have the encoded name (useful around region replicas
+ * for example which do not have a row in hbase:meta).
+ */
@Override
@QosPriority(priority=HConstants.ADMIN_QOS)
public GetRegionInfoResponse getRegionInfo(final RpcController controller,
final GetRegionInfoRequest request) throws ServiceException {
- byte[] regionName = request.getRegion().getValue().toByteArray();
- TableName tableName = RegionInfo.getTable(regionName);
- if (MobUtils.isMobRegionName(tableName, regionName)) {
- // a dummy region info contains the compaction state.
- RegionInfo mobRegionInfo = MobUtils.getMobRegionInfo(tableName);
- GetRegionInfoResponse.Builder builder = GetRegionInfoResponse.newBuilder();
- builder.setRegionInfo(ProtobufUtil.toRegionInfo(mobRegionInfo));
- if (request.hasCompactionState() && request.getCompactionState()) {
- builder.setCompactionState(master.getMobCompactionState(tableName));
- }
- return builder.build();
+ RegionInfo ri = null;
+ try {
+ ri = getRegionInfo(request.getRegion());
+ } catch(UnknownRegionException ure) {
+ throw new ServiceException(ure);
+ }
+ GetRegionInfoResponse.Builder builder = GetRegionInfoResponse.newBuilder();
+ if (ri != null) {
+ builder.setRegionInfo(ProtobufUtil.toRegionInfo(ri));
} else {
- return super.getRegionInfo(controller, request);
+ // Is it a MOB name? These work differently.
+ byte [] regionName = request.getRegion().getValue().toByteArray();
+ TableName tableName = RegionInfo.getTable(regionName);
+ if (MobUtils.isMobRegionName(tableName, regionName)) {
+ // a dummy region info contains the compaction state.
+ RegionInfo mobRegionInfo = MobUtils.getMobRegionInfo(tableName);
+ builder.setRegionInfo(ProtobufUtil.toRegionInfo(mobRegionInfo));
+ if (request.hasCompactionState() && request.getCompactionState()) {
+ builder.setCompactionState(master.getMobCompactionState(tableName));
+ }
+ } else {
+ // If unknown RegionInfo and not a MOB region, it is unknown.
+ throw new ServiceException(new UnknownRegionException(Bytes.toString(regionName)));
+ }
}
+ return builder.build();
}
/**
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
index 0b1fd0b..7f41f78 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
@@ -1756,6 +1756,9 @@ public class RSRpcServices implements HBaseRPCErrorHandler,
}
}
+ // Master implementation of this Admin Service differs given it is not
+ // able to supply detail only known to RegionServer. See note on
+ // MasterRpcServers#getRegionInfo.
@Override
@QosPriority(priority=HConstants.ADMIN_QOS)
public GetRegionInfoResponse getRegionInfo(final RpcController controller,
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin2.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin2.java
index 9cbb474..90a73ce 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin2.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin2.java
@@ -793,6 +793,34 @@ public class TestAdmin2 {
// Make sure that the store size is still the actual file system's store size.
Assert.assertEquals(expectedStoreFilesSize, store.getSize());
}
+
+ // Test querying using the encoded name only. When encoded name passed,
+ // and the target server is the Master, we return the full region name.
+ // Convenience.
+ testGetWithEncodedRegionName(conn, region.getRegionInfo());
+ testGetWithRegionName(conn, region.getRegionInfo());
+ // Try querying meta encoded name.
+ testGetWithEncodedRegionName(conn, RegionInfoBuilder.FIRST_META_REGIONINFO);
+ testGetWithRegionName(conn, RegionInfoBuilder.FIRST_META_REGIONINFO);
+ }
+
+ /**
+ * Do get of RegionInfo from Master using encoded region name.
+ */
+ private void testGetWithEncodedRegionName(ClusterConnection conn, RegionInfo inputRI)
+ throws IOException {
+ RegionInfo ri = ProtobufUtil.getRegionInfo(null,
+ conn.getAdmin(TEST_UTIL.getMiniHBaseCluster().getMaster().getServerName()),
+ inputRI.getEncodedNameAsBytes());
+ assertEquals(inputRI, ri);
+ }
+
+ private void testGetWithRegionName(ClusterConnection conn, RegionInfo inputRI)
+ throws IOException {
+ RegionInfo ri = ProtobufUtil.getRegionInfo(null,
+ conn.getAdmin(TEST_UTIL.getMiniHBaseCluster().getMaster().getServerName()),
+ inputRI.getRegionName());
+ assertEquals(inputRI, ri);
}
@Test
diff --git a/hbase-shell/src/main/ruby/shell.rb b/hbase-shell/src/main/ruby/shell.rb
index 3c6f48f..0065a4b 100644
--- a/hbase-shell/src/main/ruby/shell.rb
+++ b/hbase-shell/src/main/ruby/shell.rb
@@ -362,6 +362,7 @@ Shell.load_command_group(
clear_block_cache
stop_master
stop_regionserver
+ regioninfo
rit
list_decommissioned_regionservers
decommission_regionservers
diff --git a/hbase-shell/src/main/ruby/shell/commands/regioninfo.rb b/hbase-shell/src/main/ruby/shell/commands/regioninfo.rb
new file mode 100644
index 0000000..c611705
--- /dev/null
+++ b/hbase-shell/src/main/ruby/shell/commands/regioninfo.rb
@@ -0,0 +1,47 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module Shell
+ module Commands
+ class Regioninfo < Command
+ def help
+ <<-EOF
+Return RegionInfo. Takes Region name or an encoded Region name
+(Of use when all you have is an encoded Region name).
+
+Examples:
+Below we pass first encoded region name and then full region name.
+
+ hbase(main):002:0> regioninfo '1588230740'
+ {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
+ hbase(main):002:0> regioninfo 'hbase:meta,,1'
+ {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
+
+EOF
+ end
+
+ def command(regionname)
+ connection = org.apache.hadoop.hbase.client.ConnectionFactory.createConnection()
+ admin = connection.getAdmin()
+ sn = servername != nil ? ServerName.valueOf(servername): admin.getMaster()
+ puts org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRegionInfo(nil,
+ connection.getAdmin(sn), regionname.to_java_bytes)
+ end
+ end
+ end
+end