You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zh...@apache.org on 2019/01/23 10:14:18 UTC
[hbase] branch master updated: HBASE-21753 Support getting the
locations for all the replicas of a region
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new dfad304 HBASE-21753 Support getting the locations for all the replicas of a region
dfad304 is described below
commit dfad304ddb4c5e42ddb5b924ac6b9c6cb568439c
Author: Duo Zhang <zh...@apache.org>
AuthorDate: Tue Jan 22 16:56:47 2019 +0800
HBASE-21753 Support getting the locations for all the replicas of a region
---
.../apache/hadoop/hbase/client/HRegionLocator.java | 84 +++--------
.../org/apache/hadoop/hbase/client/HTable.java | 2 +-
.../apache/hadoop/hbase/client/RegionLocator.java | 93 ++++++++++--
.../hadoop/hbase/client/TestFromClientSide.java | 15 +-
.../hadoop/hbase/client/TestRegionLocator.java | 166 +++++++++++++++++++++
5 files changed, 279 insertions(+), 81 deletions(-)
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HRegionLocator.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HRegionLocator.java
index 2559114..3827d31 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HRegionLocator.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HRegionLocator.java
@@ -20,36 +20,29 @@ package org.apache.hadoop.hbase.client;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.MetaTableAccessor;
import org.apache.hadoop.hbase.RegionLocations;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.util.Pair;
import org.apache.yetus.audience.InterfaceAudience;
-import org.apache.yetus.audience.InterfaceStability;
-
-import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
/**
* An implementation of {@link RegionLocator}. Used to view region location information for a single
* HBase table. Lightweight. Get as needed and just close when done. Instances of this class SHOULD
* NOT be constructed directly. Obtain an instance via {@link Connection}. See
* {@link ConnectionFactory} class comment for an example of how.
- *
- * <p> This class is thread safe
+ * <p/>
+ * This class is thread safe
*/
@InterfaceAudience.Private
-@InterfaceStability.Stable
public class HRegionLocator implements RegionLocator {
private final TableName tableName;
- private final ClusterConnection connection;
+ private final ConnectionImplementation connection;
- public HRegionLocator(TableName tableName, ClusterConnection connection) {
+ public HRegionLocator(TableName tableName, ConnectionImplementation connection) {
this.connection = connection;
this.tableName = tableName;
}
@@ -63,22 +56,18 @@ public class HRegionLocator implements RegionLocator {
// persistent state, so there is no need to do anything here.
}
- /**
- * {@inheritDoc}
- */
@Override
- public HRegionLocation getRegionLocation(final byte [] row)
- throws IOException {
- return connection.getRegionLocation(tableName, row, false);
+ public HRegionLocation getRegionLocation(byte[] row, int replicaId, boolean reload)
+ throws IOException {
+ return connection.locateRegion(tableName, row, !reload, true, replicaId)
+ .getRegionLocation(replicaId);
}
- /**
- * {@inheritDoc}
- */
@Override
- public HRegionLocation getRegionLocation(final byte [] row, boolean reload)
- throws IOException {
- return connection.getRegionLocation(tableName, row, reload);
+ public List<HRegionLocation> getRegionLocations(byte[] row, boolean reload) throws IOException {
+ RegionLocations locs =
+ connection.locateRegion(tableName, row, !reload, true, RegionInfo.DEFAULT_REPLICA_ID);
+ return Arrays.asList(locs.getRegionLocations());
}
@Override
@@ -94,42 +83,9 @@ public class HRegionLocator implements RegionLocator {
return regions;
}
- /**
- * {@inheritDoc}
- */
@Override
- public byte[][] getStartKeys() throws IOException {
- return getStartEndKeys().getFirst();
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public byte[][] getEndKeys() throws IOException {
- return getStartEndKeys().getSecond();
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public Pair<byte[][], byte[][]> getStartEndKeys() throws IOException {
- return getStartEndKeys(listRegionLocations());
- }
-
- @VisibleForTesting
- Pair<byte[][], byte[][]> getStartEndKeys(List<RegionLocations> regions) {
- final byte[][] startKeyList = new byte[regions.size()][];
- final byte[][] endKeyList = new byte[regions.size()][];
-
- for (int i = 0; i < regions.size(); i++) {
- HRegionInfo region = regions.get(i).getRegionLocation().getRegionInfo();
- startKeyList[i] = region.getStartKey();
- endKeyList[i] = region.getEndKey();
- }
-
- return new Pair<>(startKeyList, endKeyList);
+ public void clearRegionLocationCache() {
+ connection.clearRegionCache(tableName);
}
@Override
@@ -137,14 +93,15 @@ public class HRegionLocator implements RegionLocator {
return this.tableName;
}
- @VisibleForTesting
- List<RegionLocations> listRegionLocations() throws IOException {
+ private List<RegionLocations> listRegionLocations() throws IOException {
final List<RegionLocations> regions = new ArrayList<>();
MetaTableAccessor.Visitor visitor = new MetaTableAccessor.TableVisitorBase(tableName) {
@Override
public boolean visitInternal(Result result) throws IOException {
RegionLocations locations = MetaTableAccessor.getRegionLocations(result);
- if (locations == null) return true;
+ if (locations == null) {
+ return true;
+ }
regions.add(locations);
return true;
}
@@ -153,7 +110,4 @@ public class HRegionLocator implements RegionLocator {
return regions;
}
- public Configuration getConfiguration() {
- return connection.getConfiguration();
- }
}
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTable.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTable.java
index 4b86e22..15a189c 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTable.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTable.java
@@ -157,7 +157,7 @@ public class HTable implements Table {
* @param pool ExecutorService to be used.
*/
@InterfaceAudience.Private
- protected HTable(final ClusterConnection connection,
+ protected HTable(final ConnectionImplementation connection,
final TableBuilderBase builder,
final RpcRetryingCallerFactory rpcCallerFactory,
final RpcControllerFactory rpcControllerFactory,
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionLocator.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionLocator.java
index a44f739..fbea0f5 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionLocator.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionLocator.java
@@ -21,16 +21,15 @@ package org.apache.hadoop.hbase.client;
import java.io.Closeable;
import java.io.IOException;
import java.util.List;
-
+import java.util.stream.Collectors;
import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.TableName;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hadoop.hbase.util.Pair;
/**
- * Used to view region location information for a single HBase table.
- * Obtain an instance from an {@link Connection}.
- *
+ * Used to view region location information for a single HBase table. Obtain an instance from an
+ * {@link Connection}.
* @see ConnectionFactory
* @see Connection
* @see Table
@@ -44,7 +43,9 @@ public interface RegionLocator extends Closeable {
* @return Location of the row.
* @throws IOException if a remote or network exception occurs
*/
- public HRegionLocation getRegionLocation(final byte [] row) throws IOException;
+ default HRegionLocation getRegionLocation(byte[] row) throws IOException {
+ return getRegionLocation(row, false);
+ }
/**
* Finds the region on which the given row is being served.
@@ -53,16 +54,65 @@ public interface RegionLocator extends Closeable {
* @return Location of the row.
* @throws IOException if a remote or network exception occurs
*/
- public HRegionLocation getRegionLocation(final byte [] row, boolean reload)
- throws IOException;
+ default HRegionLocation getRegionLocation(byte[] row, boolean reload) throws IOException {
+ return getRegionLocation(row, RegionInfo.DEFAULT_REPLICA_ID, reload);
+ }
+
+ /**
+ * Finds the region with the given replica id on which the given row is being served.
+ * @param row Row to find.
+ * @param replicaId the replica id
+ * @return Location of the row.
+ * @throws IOException if a remote or network exception occurs
+ */
+ default HRegionLocation getRegionLocation(byte[] row, int replicaId) throws IOException {
+ return getRegionLocation(row, replicaId, false);
+ }
+
+ /**
+ * Finds the region with the given replica id on which the given row is being served.
+ * @param row Row to find.
+ * @param replicaId the replica id
+ * @param reload true to reload information or false to use cached information
+ * @return Location of the row.
+ * @throws IOException if a remote or network exception occurs
+ */
+ HRegionLocation getRegionLocation(byte[] row, int replicaId, boolean reload) throws IOException;
+
+ /**
+ * Find all the replicas for the region on which the given row is being served.
+ * @param row Row to find.
+ * @return Locations for all the replicas of the row.
+ * @throws IOException if a remote or network exception occurs
+ */
+ default List<HRegionLocation> getRegionLocations(byte[] row) throws IOException {
+ return getRegionLocations(row, false);
+ }
+
+ /**
+ * Find all the replicas for the region on which the given row is being served.
+ * @param row Row to find.
+ * @param reload true to reload information or false to use cached information
+ * @return Locations for all the replicas of the row.
+ * @throws IOException if a remote or network exception occurs
+ */
+ List<HRegionLocation> getRegionLocations(byte[] row, boolean reload) throws IOException;
+
+ /**
+ * Clear all the entries in the region location cache.
+ * <p/>
+ * This may cause performance issue so use it with caution.
+ */
+ void clearRegionLocationCache();
/**
* Retrieves all of the regions associated with this table.
+ * <p/>
+ * Notice that the location for region replicas other than the default replica are also returned.
* @return a {@link List} of all regions associated with this table.
* @throws IOException if a remote or network exception occurs
*/
- public List<HRegionLocation> getAllRegionLocations()
- throws IOException;
+ List<HRegionLocation> getAllRegionLocations() throws IOException;
/**
* Gets the starting row key for every region in the currently open table.
@@ -71,7 +121,9 @@ public interface RegionLocator extends Closeable {
* @return Array of region starting row keys
* @throws IOException if a remote or network exception occurs
*/
- public byte [][] getStartKeys() throws IOException;
+ default byte[][] getStartKeys() throws IOException {
+ return getStartEndKeys().getFirst();
+ }
/**
* Gets the ending row key for every region in the currently open table.
@@ -80,17 +132,30 @@ public interface RegionLocator extends Closeable {
* @return Array of region ending row keys
* @throws IOException if a remote or network exception occurs
*/
- public byte[][] getEndKeys() throws IOException;
+ default byte[][] getEndKeys() throws IOException {
+ return getStartEndKeys().getSecond();
+ }
/**
- * Gets the starting and ending row keys for every region in the currently
- * open table.
+ * Gets the starting and ending row keys for every region in the currently open table.
* <p>
* This is mainly useful for the MapReduce integration.
* @return Pair of arrays of region starting and ending row keys
* @throws IOException if a remote or network exception occurs
*/
- public Pair<byte[][],byte[][]> getStartEndKeys() throws IOException;
+ default Pair<byte[][], byte[][]> getStartEndKeys() throws IOException {
+ List<HRegionLocation> regions = getAllRegionLocations().stream()
+ .filter(loc -> RegionReplicaUtil.isDefaultReplica(loc.getRegion()))
+ .collect(Collectors.toList());
+ byte[][] startKeys = new byte[regions.size()][];
+ byte[][] endKeys = new byte[regions.size()][];
+ for (int i = 0, n = regions.size(); i < n; i++) {
+ RegionInfo region = regions.get(i).getRegion();
+ startKeys[i] = region.getStartKey();
+ endKeys[i] = region.getEndKey();
+ }
+ return Pair.newPair(startKeys, endKeys);
+ }
/**
* Gets the fully qualified table name instance of this table.
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
index 21e4437..d9432e4 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
@@ -6352,6 +6352,19 @@ public class TestFromClientSide {
assertEquals(4, count); // 003 004 005 006
}
+ private static Pair<byte[][], byte[][]> getStartEndKeys(List<RegionLocations> regions) {
+ final byte[][] startKeyList = new byte[regions.size()][];
+ final byte[][] endKeyList = new byte[regions.size()][];
+
+ for (int i = 0; i < regions.size(); i++) {
+ RegionInfo region = regions.get(i).getRegionLocation().getRegion();
+ startKeyList[i] = region.getStartKey();
+ endKeyList[i] = region.getEndKey();
+ }
+
+ return new Pair<>(startKeyList, endKeyList);
+ }
+
@Test
public void testGetStartEndKeysWithRegionReplicas() throws IOException {
HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name.getMethodName()));
@@ -6376,7 +6389,7 @@ public class TestFromClientSide {
regionLocations.add(new RegionLocations(arr));
}
- Pair<byte[][], byte[][]> startEndKeys = locator.getStartEndKeys(regionLocations);
+ Pair<byte[][], byte[][]> startEndKeys = getStartEndKeys(regionLocations);
assertEquals(KEYS.length + 1, startEndKeys.getFirst().length);
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRegionLocator.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRegionLocator.java
new file mode 100644
index 0000000..e0634ae
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRegionLocator.java
@@ -0,0 +1,166 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionLocation;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.regionserver.Region;
+import org.apache.hadoop.hbase.testclassification.ClientTests;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category({ MediumTests.class, ClientTests.class })
+public class TestRegionLocator {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestRegionLocator.class);
+
+ private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
+
+ private static TableName TABLE_NAME = TableName.valueOf("Locator");
+
+ private static byte[] FAMILY = Bytes.toBytes("family");
+
+ private static int REGION_REPLICATION = 3;
+
+ private static byte[][] SPLIT_KEYS;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ UTIL.startMiniCluster(3);
+ TableDescriptor td =
+ TableDescriptorBuilder.newBuilder(TABLE_NAME).setRegionReplication(REGION_REPLICATION)
+ .setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY)).build();
+ SPLIT_KEYS = new byte[9][];
+ for (int i = 0; i < 9; i++) {
+ SPLIT_KEYS[i] = Bytes.toBytes(Integer.toString(i + 1));
+ }
+ UTIL.getAdmin().createTable(td, SPLIT_KEYS);
+ UTIL.waitTableAvailable(TABLE_NAME);
+ UTIL.getAdmin().balancerSwitch(false, true);
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ UTIL.shutdownMiniCluster();
+ }
+
+ @After
+ public void tearDownAfterTest() throws IOException {
+ try (RegionLocator locator = UTIL.getConnection().getRegionLocator(TABLE_NAME)) {
+ locator.clearRegionLocationCache();
+ }
+ }
+
+ private byte[] getStartKey(int index) {
+ return index == 0 ? HConstants.EMPTY_START_ROW : SPLIT_KEYS[index - 1];
+ }
+
+ private byte[] getEndKey(int index) {
+ return index == SPLIT_KEYS.length ? HConstants.EMPTY_END_ROW : SPLIT_KEYS[index];
+ }
+
+ private void assertStartKeys(byte[][] startKeys) {
+ assertEquals(SPLIT_KEYS.length + 1, startKeys.length);
+ for (int i = 0; i < startKeys.length; i++) {
+ assertArrayEquals(getStartKey(i), startKeys[i]);
+ }
+ }
+
+ private void assertEndKeys(byte[][] endKeys) {
+ assertEquals(SPLIT_KEYS.length + 1, endKeys.length);
+ for (int i = 0; i < endKeys.length; i++) {
+ assertArrayEquals(getEndKey(i), endKeys[i]);
+ }
+ }
+
+ @Test
+ public void testStartEndKeys() throws IOException {
+ try (RegionLocator locator = UTIL.getConnection().getRegionLocator(TABLE_NAME)) {
+ assertStartKeys(locator.getStartKeys());
+ assertEndKeys(locator.getEndKeys());
+ Pair<byte[][], byte[][]> startEndKeys = locator.getStartEndKeys();
+ assertStartKeys(startEndKeys.getFirst());
+ assertEndKeys(startEndKeys.getSecond());
+ }
+ }
+
+ private void assertRegionLocation(HRegionLocation loc, int index, int replicaId) {
+ RegionInfo region = loc.getRegion();
+ byte[] startKey = getStartKey(index);
+ assertArrayEquals(startKey, region.getStartKey());
+ assertArrayEquals(getEndKey(index), region.getEndKey());
+ assertEquals(replicaId, region.getReplicaId());
+ ServerName expected =
+ UTIL.getMiniHBaseCluster().getRegionServerThreads().stream().map(t -> t.getRegionServer())
+ .filter(rs -> rs.getRegions(TABLE_NAME).stream().map(Region::getRegionInfo)
+ .anyMatch(r -> r.containsRow(startKey) && r.getReplicaId() == replicaId))
+ .findFirst().get().getServerName();
+ assertEquals(expected, loc.getServerName());
+ }
+
+ @Test
+ public void testGetRegionLocation() throws IOException {
+ try (RegionLocator locator = UTIL.getConnection().getRegionLocator(TABLE_NAME)) {
+ for (int i = 0; i <= SPLIT_KEYS.length; i++) {
+ for (int replicaId = 0; replicaId < REGION_REPLICATION; replicaId++) {
+ assertRegionLocation(locator.getRegionLocation(getStartKey(i), replicaId), i, replicaId);
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testGetAllRegionLocations() throws IOException {
+ try (RegionLocator locator = UTIL.getConnection().getRegionLocator(TABLE_NAME)) {
+ List<HRegionLocation> locs = locator.getAllRegionLocations();
+ assertEquals(REGION_REPLICATION * (SPLIT_KEYS.length + 1), locs.size());
+ Collections.sort(locs, (l1, l2) -> {
+ int c = Bytes.compareTo(l1.getRegion().getStartKey(), l2.getRegion().getStartKey());
+ if (c != 0) {
+ return c;
+ }
+ return Integer.compare(l1.getRegion().getReplicaId(), l2.getRegion().getReplicaId());
+ });
+ for (int i = 0; i <= SPLIT_KEYS.length; i++) {
+ for (int replicaId = 0; replicaId < REGION_REPLICATION; replicaId++) {
+ assertRegionLocation(locs.get(i * REGION_REPLICATION + replicaId), i, replicaId);
+ }
+ }
+ }
+ }
+}