You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zh...@apache.org on 2019/01/25 03:10:29 UTC
[hbase] branch branch-2 updated: HBASE-21770 Should deal with meta
table in HRegionLocator.getAllRegionLocations
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new de31d56 HBASE-21770 Should deal with meta table in HRegionLocator.getAllRegionLocations
de31d56 is described below
commit de31d56c2d4c267d2e80ffc01a5e49e760c3a4ff
Author: zhangduo <zh...@apache.org>
AuthorDate: Thu Jan 24 22:34:46 2019 +0800
HBASE-21770 Should deal with meta table in HRegionLocator.getAllRegionLocations
Signed-off-by: Guanghao Zhang <zg...@apache.org>
---
.../apache/hadoop/hbase/client/HRegionLocator.java | 8 +-
.../hbase/client/AbstractTestRegionLocator.java | 91 +++++++++++++++++-----
.../hbase/client/TestAsyncTableRegionLocator.java | 35 +++++----
.../hadoop/hbase/client/TestRegionLocator.java | 51 +++++++-----
4 files changed, 127 insertions(+), 58 deletions(-)
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HRegionLocator.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HRegionLocator.java
index 3827d31..8806a3e3 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HRegionLocator.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HRegionLocator.java
@@ -21,7 +21,9 @@ package org.apache.hadoop.hbase.client;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
+import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.MetaTableAccessor;
import org.apache.hadoop.hbase.RegionLocations;
@@ -72,7 +74,6 @@ public class HRegionLocator implements RegionLocator {
@Override
public List<HRegionLocation> getAllRegionLocations() throws IOException {
- TableName tableName = getName();
ArrayList<HRegionLocation> regions = new ArrayList<>();
for (RegionLocations locations : listRegionLocations()) {
for (HRegionLocation location : locations.getRegionLocations()) {
@@ -94,6 +95,10 @@ public class HRegionLocator implements RegionLocator {
}
private List<RegionLocations> listRegionLocations() throws IOException {
+ if (TableName.isMetaTableName(tableName)) {
+ return Collections
+ .singletonList(connection.locateRegion(tableName, HConstants.EMPTY_START_ROW, false, true));
+ }
final List<RegionLocations> regions = new ArrayList<>();
MetaTableAccessor.Visitor visitor = new MetaTableAccessor.TableVisitorBase(tableName) {
@Override
@@ -109,5 +114,4 @@ public class HRegionLocator implements RegionLocator {
MetaTableAccessor.scanMetaForTableRegions(connection, visitor, tableName);
return regions;
}
-
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/AbstractTestRegionLocator.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/AbstractTestRegionLocator.java
index b21c33f..25ee41d 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/AbstractTestRegionLocator.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/AbstractTestRegionLocator.java
@@ -47,6 +47,7 @@ public abstract class AbstractTestRegionLocator {
protected static byte[][] SPLIT_KEYS;
protected static void startClusterAndCreateTable() throws Exception {
+ UTIL.getConfiguration().setInt(HConstants.META_REPLICAS_NUM, REGION_REPLICATION);
UTIL.startMiniCluster(3);
TableDescriptor td =
TableDescriptorBuilder.newBuilder(TABLE_NAME).setRegionReplication(REGION_REPLICATION)
@@ -57,12 +58,17 @@ public abstract class AbstractTestRegionLocator {
}
UTIL.getAdmin().createTable(td, SPLIT_KEYS);
UTIL.waitTableAvailable(TABLE_NAME);
+ try (AsyncRegistry registry = AsyncRegistryFactory.getRegistry(UTIL.getConfiguration())) {
+ RegionReplicaTestHelper.waitUntilAllMetaReplicasHavingRegionLocation(registry,
+ REGION_REPLICATION);
+ }
UTIL.getAdmin().balancerSwitch(false, true);
}
@After
public void tearDownAfterTest() throws IOException {
- clearCache();
+ clearCache(TABLE_NAME);
+ clearCache(TableName.META_TABLE_NAME);
}
private byte[] getStartKey(int index) {
@@ -89,9 +95,9 @@ public abstract class AbstractTestRegionLocator {
@Test
public void testStartEndKeys() throws IOException {
- assertStartKeys(getStartKeys());
- assertEndKeys(getEndKeys());
- Pair<byte[][], byte[][]> startEndKeys = getStartEndKeys();
+ assertStartKeys(getStartKeys(TABLE_NAME));
+ assertEndKeys(getEndKeys(TABLE_NAME));
+ Pair<byte[][], byte[][]> startEndKeys = getStartEndKeys(TABLE_NAME);
assertStartKeys(startEndKeys.getFirst());
assertEndKeys(startEndKeys.getSecond());
}
@@ -102,19 +108,24 @@ public abstract class AbstractTestRegionLocator {
assertArrayEquals(startKey, region.getStartKey());
assertArrayEquals(getEndKey(index), region.getEndKey());
assertEquals(replicaId, region.getReplicaId());
- ServerName expected =
- UTIL.getMiniHBaseCluster().getRegionServerThreads().stream().map(t -> t.getRegionServer())
- .filter(rs -> rs.getRegions(TABLE_NAME).stream().map(Region::getRegionInfo)
- .anyMatch(r -> r.containsRow(startKey) && r.getReplicaId() == replicaId))
- .findFirst().get().getServerName();
+ ServerName expected = findRegionLocation(TABLE_NAME, region.getStartKey(), replicaId);
assertEquals(expected, loc.getServerName());
}
+ private ServerName findRegionLocation(TableName tableName, byte[] startKey, int replicaId) {
+ return UTIL.getMiniHBaseCluster().getRegionServerThreads().stream()
+ .map(t -> t.getRegionServer())
+ .filter(rs -> rs.getRegions(tableName).stream().map(Region::getRegionInfo)
+ .anyMatch(r -> r.containsRow(startKey) && r.getReplicaId() == replicaId))
+ .findFirst().get().getServerName();
+ }
+
@Test
public void testGetRegionLocation() throws IOException {
for (int i = 0; i <= SPLIT_KEYS.length; i++) {
for (int replicaId = 0; replicaId < REGION_REPLICATION; replicaId++) {
- assertRegionLocation(getRegionLocation(getStartKey(i), replicaId), i, replicaId);
+ assertRegionLocation(getRegionLocation(TABLE_NAME, getStartKey(i), replicaId), i,
+ replicaId);
}
}
}
@@ -122,7 +133,7 @@ public abstract class AbstractTestRegionLocator {
@Test
public void testGetRegionLocations() throws IOException {
for (int i = 0; i <= SPLIT_KEYS.length; i++) {
- List<HRegionLocation> locs = getRegionLocations(getStartKey(i));
+ List<HRegionLocation> locs = getRegionLocations(TABLE_NAME, getStartKey(i));
assertEquals(REGION_REPLICATION, locs.size());
for (int replicaId = 0; replicaId < REGION_REPLICATION; replicaId++) {
assertRegionLocation(locs.get(replicaId), i, replicaId);
@@ -132,7 +143,7 @@ public abstract class AbstractTestRegionLocator {
@Test
public void testGetAllRegionLocations() throws IOException {
- List<HRegionLocation> locs = getAllRegionLocations();
+ List<HRegionLocation> locs = getAllRegionLocations(TABLE_NAME);
assertEquals(REGION_REPLICATION * (SPLIT_KEYS.length + 1), locs.size());
Collections.sort(locs, (l1, l2) -> {
int c = Bytes.compareTo(l1.getRegion().getStartKey(), l2.getRegion().getStartKey());
@@ -148,18 +159,60 @@ public abstract class AbstractTestRegionLocator {
}
}
- protected abstract byte[][] getStartKeys() throws IOException;
+ private void assertMetaStartOrEndKeys(byte[][] keys) {
+ assertEquals(1, keys.length);
+ assertArrayEquals(HConstants.EMPTY_BYTE_ARRAY, keys[0]);
+ }
- protected abstract byte[][] getEndKeys() throws IOException;
+ private void assertMetaRegionLocation(HRegionLocation loc, int replicaId) {
+ RegionInfo region = loc.getRegion();
+ assertArrayEquals(HConstants.EMPTY_START_ROW, region.getStartKey());
+ assertArrayEquals(HConstants.EMPTY_END_ROW, region.getEndKey());
+ assertEquals(replicaId, region.getReplicaId());
+ ServerName expected =
+ findRegionLocation(TableName.META_TABLE_NAME, region.getStartKey(), replicaId);
+ assertEquals(expected, loc.getServerName());
+ }
- protected abstract Pair<byte[][], byte[][]> getStartEndKeys() throws IOException;
+ private void assertMetaRegionLocations(List<HRegionLocation> locs) {
+ assertEquals(REGION_REPLICATION, locs.size());
+ for (int replicaId = 0; replicaId < REGION_REPLICATION; replicaId++) {
+ assertMetaRegionLocation(locs.get(replicaId), replicaId);
+ }
+ }
+
+ @Test
+ public void testMeta() throws IOException {
+ assertMetaStartOrEndKeys(getStartKeys(TableName.META_TABLE_NAME));
+ assertMetaStartOrEndKeys(getEndKeys(TableName.META_TABLE_NAME));
+ Pair<byte[][], byte[][]> startEndKeys = getStartEndKeys(TableName.META_TABLE_NAME);
+ assertMetaStartOrEndKeys(startEndKeys.getFirst());
+ assertMetaStartOrEndKeys(startEndKeys.getSecond());
+ for (int replicaId = 0; replicaId < REGION_REPLICATION; replicaId++) {
+ assertMetaRegionLocation(
+ getRegionLocation(TableName.META_TABLE_NAME, HConstants.EMPTY_START_ROW, replicaId),
+ replicaId);
+ }
+ assertMetaRegionLocations(
+ getRegionLocations(TableName.META_TABLE_NAME, HConstants.EMPTY_START_ROW));
+ assertMetaRegionLocations(getAllRegionLocations(TableName.META_TABLE_NAME));
+ }
- protected abstract HRegionLocation getRegionLocation(byte[] row, int replicaId)
+ protected abstract byte[][] getStartKeys(TableName tableName) throws IOException;
+
+ protected abstract byte[][] getEndKeys(TableName tableName) throws IOException;
+
+ protected abstract Pair<byte[][], byte[][]> getStartEndKeys(TableName tableName)
throws IOException;
- protected abstract List<HRegionLocation> getRegionLocations(byte[] row) throws IOException;
+ protected abstract HRegionLocation getRegionLocation(TableName tableName, byte[] row,
+ int replicaId) throws IOException;
- protected abstract List<HRegionLocation> getAllRegionLocations() throws IOException;
+ protected abstract List<HRegionLocation> getRegionLocations(TableName tableName, byte[] row)
+ throws IOException;
+
+ protected abstract List<HRegionLocation> getAllRegionLocations(TableName tableName)
+ throws IOException;
- protected abstract void clearCache() throws IOException;
+ protected abstract void clearCache(TableName tableName) throws IOException;
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableRegionLocator.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableRegionLocator.java
index f32693a..29865e7 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableRegionLocator.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableRegionLocator.java
@@ -23,6 +23,7 @@ import java.io.IOException;
import java.util.List;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HRegionLocation;
+import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.testclassification.ClientTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.Pair;
@@ -42,13 +43,10 @@ public class TestAsyncTableRegionLocator extends AbstractTestRegionLocator {
private static AsyncConnection CONN;
- private static AsyncTableRegionLocator LOCATOR;
-
@BeforeClass
public static void setUp() throws Exception {
startClusterAndCreateTable();
CONN = ConnectionFactory.createAsyncConnection(UTIL.getConfiguration()).get();
- LOCATOR = CONN.getRegionLocator(TABLE_NAME);
}
@AfterClass
@@ -58,18 +56,19 @@ public class TestAsyncTableRegionLocator extends AbstractTestRegionLocator {
}
@Override
- protected byte[][] getStartKeys() throws IOException {
- return get(LOCATOR.getStartKeys()).toArray(new byte[0][]);
+ protected byte[][] getStartKeys(TableName tableName) throws IOException {
+ return get(CONN.getRegionLocator(tableName).getStartKeys()).toArray(new byte[0][]);
}
@Override
- protected byte[][] getEndKeys() throws IOException {
- return get(LOCATOR.getEndKeys()).toArray(new byte[0][]);
+ protected byte[][] getEndKeys(TableName tableName) throws IOException {
+ return get(CONN.getRegionLocator(tableName).getEndKeys()).toArray(new byte[0][]);
}
@Override
- protected Pair<byte[][], byte[][]> getStartEndKeys() throws IOException {
- List<Pair<byte[], byte[]>> startEndKeys = get(LOCATOR.getStartEndKeys());
+ protected Pair<byte[][], byte[][]> getStartEndKeys(TableName tableName) throws IOException {
+ List<Pair<byte[], byte[]>> startEndKeys =
+ get(CONN.getRegionLocator(tableName).getStartEndKeys());
byte[][] startKeys = new byte[startEndKeys.size()][];
byte[][] endKeys = new byte[startEndKeys.size()][];
for (int i = 0, n = startEndKeys.size(); i < n; i++) {
@@ -81,22 +80,24 @@ public class TestAsyncTableRegionLocator extends AbstractTestRegionLocator {
}
@Override
- protected HRegionLocation getRegionLocation(byte[] row, int replicaId) throws IOException {
- return get(LOCATOR.getRegionLocation(row, replicaId));
+ protected HRegionLocation getRegionLocation(TableName tableName, byte[] row, int replicaId)
+ throws IOException {
+ return get(CONN.getRegionLocator(tableName).getRegionLocation(row, replicaId));
}
@Override
- protected List<HRegionLocation> getRegionLocations(byte[] row) throws IOException {
- return get(LOCATOR.getRegionLocations(row));
+ protected List<HRegionLocation> getRegionLocations(TableName tableName, byte[] row)
+ throws IOException {
+ return get(CONN.getRegionLocator(tableName).getRegionLocations(row));
}
@Override
- protected List<HRegionLocation> getAllRegionLocations() throws IOException {
- return get(LOCATOR.getAllRegionLocations());
+ protected List<HRegionLocation> getAllRegionLocations(TableName tableName) throws IOException {
+ return get(CONN.getRegionLocator(tableName).getAllRegionLocations());
}
@Override
- protected void clearCache() throws IOException {
- LOCATOR.clearRegionLocationCache();
+ protected void clearCache(TableName tableName) throws IOException {
+ CONN.getRegionLocator(tableName).clearRegionLocationCache();
}
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRegionLocator.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRegionLocator.java
index 49ce8b1..081d848 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRegionLocator.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRegionLocator.java
@@ -21,6 +21,7 @@ import java.io.IOException;
import java.util.List;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HRegionLocation;
+import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.testclassification.ClientTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.Pair;
@@ -29,8 +30,6 @@ import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.experimental.categories.Category;
-import org.apache.hbase.thirdparty.com.google.common.io.Closeables;
-
@Category({ MediumTests.class, ClientTests.class })
public class TestRegionLocator extends AbstractTestRegionLocator {
@@ -38,52 +37,64 @@ public class TestRegionLocator extends AbstractTestRegionLocator {
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestRegionLocator.class);
- private static RegionLocator LOCATOR;
-
@BeforeClass
public static void setUp() throws Exception {
startClusterAndCreateTable();
- LOCATOR = UTIL.getConnection().getRegionLocator(TABLE_NAME);
}
@AfterClass
public static void tearDown() throws Exception {
- Closeables.close(LOCATOR, true);
UTIL.shutdownMiniCluster();
}
@Override
- protected byte[][] getStartKeys() throws IOException {
- return LOCATOR.getStartKeys();
+ protected byte[][] getStartKeys(TableName tableName) throws IOException {
+ try (RegionLocator locator = UTIL.getConnection().getRegionLocator(tableName)) {
+ return locator.getStartKeys();
+ }
}
@Override
- protected byte[][] getEndKeys() throws IOException {
- return LOCATOR.getEndKeys();
+ protected byte[][] getEndKeys(TableName tableName) throws IOException {
+ try (RegionLocator locator = UTIL.getConnection().getRegionLocator(tableName)) {
+ return locator.getEndKeys();
+ }
}
@Override
- protected Pair<byte[][], byte[][]> getStartEndKeys() throws IOException {
- return LOCATOR.getStartEndKeys();
+ protected Pair<byte[][], byte[][]> getStartEndKeys(TableName tableName) throws IOException {
+ try (RegionLocator locator = UTIL.getConnection().getRegionLocator(tableName)) {
+ return locator.getStartEndKeys();
+ }
}
@Override
- protected HRegionLocation getRegionLocation(byte[] row, int replicaId) throws IOException {
- return LOCATOR.getRegionLocation(row, replicaId);
+ protected HRegionLocation getRegionLocation(TableName tableName, byte[] row, int replicaId)
+ throws IOException {
+ try (RegionLocator locator = UTIL.getConnection().getRegionLocator(tableName)) {
+ return locator.getRegionLocation(row, replicaId);
+ }
}
@Override
- protected List<HRegionLocation> getRegionLocations(byte[] row) throws IOException {
- return LOCATOR.getRegionLocations(row);
+ protected List<HRegionLocation> getRegionLocations(TableName tableName, byte[] row)
+ throws IOException {
+ try (RegionLocator locator = UTIL.getConnection().getRegionLocator(tableName)) {
+ return locator.getRegionLocations(row);
+ }
}
@Override
- protected List<HRegionLocation> getAllRegionLocations() throws IOException {
- return LOCATOR.getAllRegionLocations();
+ protected List<HRegionLocation> getAllRegionLocations(TableName tableName) throws IOException {
+ try (RegionLocator locator = UTIL.getConnection().getRegionLocator(tableName)) {
+ return locator.getAllRegionLocations();
+ }
}
@Override
- protected void clearCache() throws IOException {
- LOCATOR.clearRegionLocationCache();
+ protected void clearCache(TableName tableName) throws IOException {
+ try (RegionLocator locator = UTIL.getConnection().getRegionLocator(tableName)) {
+ locator.clearRegionLocationCache();
+ }
}
}