You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2018/02/24 01:55:24 UTC
[1/5] hbase git commit: HBASE-19391 Calling
HRegion#initializeRegionInternals from a region replica can still re-create a
region directory
Repository: hbase
Updated Branches:
refs/heads/branch-1.4 20cb47c38 -> 0d8fee215
HBASE-19391 Calling HRegion#initializeRegionInternals from a region replica can still re-create a region directory
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2d12ae20
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2d12ae20
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2d12ae20
Branch: refs/heads/branch-1.4
Commit: 2d12ae2031e5d925e24b2bcf2005bbbd5ef7844e
Parents: 20cb47c
Author: Esteban Gutierrez <es...@apache.org>
Authored: Wed Feb 21 15:28:50 2018 -0800
Committer: Andrew Purtell <ap...@apache.org>
Committed: Fri Feb 23 16:57:56 2018 -0800
----------------------------------------------------------------------
.../hbase/regionserver/HRegionFileSystem.java | 29 ++++++++++----------
1 file changed, 15 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/2d12ae20/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
index 884485c..1a66063 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
@@ -950,28 +950,29 @@ public class HRegionFileSystem {
public static HRegionFileSystem createRegionOnFileSystem(final Configuration conf,
final FileSystem fs, final Path tableDir, final HRegionInfo regionInfo) throws IOException {
HRegionFileSystem regionFs = new HRegionFileSystem(conf, fs, tableDir, regionInfo);
- Path regionDir = regionFs.getRegionDir();
- if (fs.exists(regionDir)) {
- LOG.warn("Trying to create a region that already exists on disk: " + regionDir);
- throw new IOException("The specified region already exists on disk: " + regionDir);
- }
+ // We only create a .regioninfo and the region directory if this is the default region replica
+ if (regionInfo.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
+ Path regionDir = regionFs.getRegionDir();
+ if (fs.exists(regionDir)) {
+ LOG.warn("Trying to create a region that already exists on disk: " + regionDir);
+ throw new IOException("The specified region already exists on disk: " + regionDir);
+ }
- // Create the region directory
- if (!createDirOnFileSystem(fs, conf, regionDir)) {
- LOG.warn("Unable to create the region directory: " + regionDir);
- throw new IOException("Unable to create region directory: " + regionDir);
- }
+ // Create the region directory
+ if (!createDirOnFileSystem(fs, conf, regionDir)) {
+ LOG.warn("Unable to create the region directory: " + regionDir);
+ throw new IOException("Unable to create region directory: " + regionDir);
+ }
- // Write HRI to a file in case we need to recover hbase:meta
- // Only primary replicas should write region info
- if (regionInfo.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
+ // Write HRI to a file in case we need to recover hbase:meta
regionFs.writeRegionInfoOnFilesystem(false);
+ return regionFs;
} else {
if (LOG.isDebugEnabled())
LOG.debug("Skipping creation of .regioninfo file for " + regionInfo);
}
- return regionFs;
+ return null;
}
/**
[4/5] hbase git commit: HBASE-19364,
Truncate_preserve fails with table when replica region > 1
Posted by ap...@apache.org.
HBASE-19364, Truncate_preserve fails with table when replica region > 1
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0d8fee21
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0d8fee21
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0d8fee21
Branch: refs/heads/branch-1.4
Commit: 0d8fee2158e08bc6d0907d4abbe1215eaded6ce3
Parents: 1b0b8d9
Author: Pankaj Kumar <pa...@huawei.com>
Authored: Thu Dec 7 22:51:01 2017 +0530
Committer: Andrew Purtell <ap...@apache.org>
Committed: Fri Feb 23 17:03:03 2018 -0800
----------------------------------------------------------------------
.../apache/hadoop/hbase/MetaTableAccessor.java | 45 +++++++++++++++++---
.../master/procedure/ProcedureSyncWait.java | 24 ++++++-----
.../procedure/TruncateTableProcedure.java | 2 +-
.../MasterProcedureTestingUtility.java | 5 +++
.../procedure/TestTruncateTableProcedure.java | 44 +++++++++++++------
5 files changed, 90 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/0d8fee21/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
index 3f11558..440f8c6 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
@@ -392,12 +392,27 @@ public class MetaTableAccessor {
* @return Ordered list of {@link HRegionInfo}.
* @throws IOException
*/
- public static List<HRegionInfo> getTableRegions(ZooKeeperWatcher zkw,
- Connection connection, TableName tableName, final boolean excludeOfflinedSplitParents)
- throws IOException {
+ public static List<HRegionInfo> getTableRegions(ZooKeeperWatcher zkw, Connection connection,
+ TableName tableName, final boolean excludeOfflinedSplitParents) throws IOException {
+ return getTableRegions(zkw, connection, tableName, excludeOfflinedSplitParents, false);
+ }
+
+ /**
+ * Gets all of the regions of the specified table.
+ * @param zkw zookeeper connection to access meta table
+ * @param connection connection we're using
+ * @param tableName table we're looking for
+ * @param excludeOfflinedSplitParents If true, do not include offlined split parents in the
+ * return.
+ * @param excludeReplicaRegions If true, do not include replica regions in the result.
+ * @return Ordered list of {@link HRegionInfo}.
+ */
+ public static List<HRegionInfo> getTableRegions(ZooKeeperWatcher zkw, Connection connection,
+ TableName tableName, final boolean excludeOfflinedSplitParents,
+ final boolean excludeReplicaRegions) throws IOException {
List<Pair<HRegionInfo, ServerName>> result = null;
- result = getTableRegionsAndLocations(zkw, connection, tableName,
- excludeOfflinedSplitParents);
+ result = getTableRegionsAndLocations(zkw, connection, tableName, excludeOfflinedSplitParents,
+ excludeReplicaRegions);
return getListOfHRegionInfos(result);
}
@@ -478,7 +493,22 @@ public class MetaTableAccessor {
public static List<Pair<HRegionInfo, ServerName>> getTableRegionsAndLocations(
ZooKeeperWatcher zkw, Connection connection, final TableName tableName,
final boolean excludeOfflinedSplitParents) throws IOException {
+ return getTableRegionsAndLocations(zkw, connection, tableName, excludeOfflinedSplitParents,
+ false);
+ }
+ /**
+ * @param zkw ZooKeeperWatcher instance we're using to get hbase:meta location
+ * @param connection connection we're using
+ * @param tableName table to work with
+ * @param excludeOfflinedSplitParents Exclude offline regions
+ * @param excludeReplicaRegions If true, do not include replica regions in the result.
+ * @return List of regioninfos and server addresses.
+ */
+ public static List<Pair<HRegionInfo, ServerName>> getTableRegionsAndLocations(
+ ZooKeeperWatcher zkw, Connection connection, final TableName tableName,
+ final boolean excludeOfflinedSplitParents, final boolean excludeReplicaRegions)
+ throws IOException {
if (tableName.equals(TableName.META_TABLE_NAME)) {
// If meta, do a bit of special handling.
ServerName serverName = new MetaTableLocator().getMetaRegionLocation(zkw);
@@ -514,6 +544,11 @@ public class MetaTableAccessor {
}
for (HRegionLocation loc : current.getRegionLocations()) {
if (loc != null) {
+ // Exclude replica region from the result
+ if (excludeReplicaRegions
+ && !RegionReplicaUtil.isDefaultReplica(loc.getRegionInfo())) {
+ continue;
+ }
this.results.add(new Pair<HRegionInfo, ServerName>(
loc.getRegionInfo(), loc.getServerName()));
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/0d8fee21/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ProcedureSyncWait.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ProcedureSyncWait.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ProcedureSyncWait.java
index d0dea0a..1846cef 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ProcedureSyncWait.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ProcedureSyncWait.java
@@ -149,22 +149,24 @@ public final class ProcedureSyncWait {
protected static List<HRegionInfo> getRegionsFromMeta(final MasterProcedureEnv env,
final TableName tableName) throws IOException {
- return getRegionsFromMeta(env, tableName, false);
+ return getRegionsFromMeta(env, tableName, false, false);
}
protected static List<HRegionInfo> getRegionsFromMeta(final MasterProcedureEnv env,
- final TableName tableName, final boolean excludeOfflinedSplitParents) throws IOException {
+ final TableName tableName, final boolean excludeOfflinedSplitParents,
+ final boolean excludeReplicaRegions) throws IOException {
return ProcedureSyncWait.waitFor(env, "regions of table=" + tableName + " from meta",
- new ProcedureSyncWait.Predicate<List<HRegionInfo>>() {
- @Override
- public List<HRegionInfo> evaluate() throws IOException {
- if (TableName.META_TABLE_NAME.equals(tableName)) {
- return new MetaTableLocator().getMetaRegions(env.getMasterServices().getZooKeeper());
- }
- return MetaTableAccessor.getTableRegions(env.getMasterServices().getZooKeeper(),
- env.getMasterServices().getConnection(), tableName, excludeOfflinedSplitParents);
+ new ProcedureSyncWait.Predicate<List<HRegionInfo>>() {
+ @Override
+ public List<HRegionInfo> evaluate() throws IOException {
+ if (TableName.META_TABLE_NAME.equals(tableName)) {
+ return new MetaTableLocator().getMetaRegions(env.getMasterServices().getZooKeeper());
}
- });
+ return MetaTableAccessor.getTableRegions(env.getMasterServices().getZooKeeper(),
+ env.getMasterServices().getConnection(), tableName, excludeOfflinedSplitParents,
+ excludeReplicaRegions);
+ }
+ });
}
protected static void waitRegionInTransition(final MasterProcedureEnv env,
http://git-wip-us.apache.org/repos/asf/hbase/blob/0d8fee21/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java
index 735e94f..120ae4f 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java
@@ -85,7 +85,7 @@ public class TruncateTableProcedure
// TODO: Move out... in the acquireLock()
LOG.debug("waiting for '" + getTableName() + "' regions in transition");
- regions = ProcedureSyncWait.getRegionsFromMeta(env, getTableName(), true);
+ regions = ProcedureSyncWait.getRegionsFromMeta(env, getTableName(), true, true);
assert regions != null && !regions.isEmpty() : "unexpected 0 regions";
ProcedureSyncWait.waitRegionInTransition(env, regions);
http://git-wip-us.apache.org/repos/asf/hbase/blob/0d8fee21/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
index fa57b12..ff479d4 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
@@ -74,6 +74,11 @@ public class MasterProcedureTestingUtility {
public static HRegionInfo[] createTable(final ProcedureExecutor<MasterProcedureEnv> procExec,
final TableName tableName, final byte[][] splitKeys, String... family) throws IOException {
HTableDescriptor htd = createHTD(tableName, family);
+ return createTable(procExec, htd, splitKeys);
+ }
+
+ public static HRegionInfo[] createTable(final ProcedureExecutor<MasterProcedureEnv> procExec,
+ HTableDescriptor htd, final byte[][] splitKeys) throws IOException {
HRegionInfo[] regions = ModifyRegionUtils.createHRegionInfos(htd, splitKeys);
long procId = ProcedureTestingUtility.submitAndWait(procExec,
new CreateTableProcedure(procExec.getEnvironment(), htd, regions));
http://git-wip-us.apache.org/repos/asf/hbase/blob/0d8fee21/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java
index 8797a4f..47460af 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java
@@ -43,6 +43,8 @@ import org.junit.experimental.categories.Category;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import java.io.IOException;
+
@Category(MediumTests.class)
public class TestTruncateTableProcedure {
private static final Log LOG = LogFactory.getLog(TestTruncateTableProcedure.class);
@@ -241,23 +243,40 @@ public class TestTruncateTableProcedure {
return UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor();
}
- @Test(timeout=60000)
- public void testTruncateWithPreserveAfterSplit() throws Exception{
+ @Test(timeout = 60000)
+ public void testTruncateWithPreserveAfterSplit() throws Exception {
final String[] families = new String[] { "f1", "f2" };
- final byte[][] splitKeys = new byte[][] {
- Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c")
- };
+ final byte[][] splitKeys =
+ new byte[][] { Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c") };
TableName tableName = TableName.valueOf("testTruncateWithPreserveAfterSplit");
- HRegionInfo[] regions = MasterProcedureTestingUtility.createTable(
- getMasterProcedureExecutor(), tableName, splitKeys, families);
+ HRegionInfo[] regions = MasterProcedureTestingUtility.createTable(getMasterProcedureExecutor(),
+ tableName, splitKeys, families);
+ splitAndTruncate(families, splitKeys, tableName, regions);
+ }
+
+ @Test(timeout = 60000)
+ public void testTruncatePreserveWithReplicaRegionAfterSplit() throws Exception {
+ final String[] families = new String[] { "f1", "f2" };
+ final byte[][] splitKeys =
+ new byte[][] { Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c") };
+ TableName tableName = TableName.valueOf("testTruncateWithPreserveAfterSplit");
+ HTableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, families);
+ htd.setRegionReplication(3);
+ HRegionInfo[] regions =
+ MasterProcedureTestingUtility.createTable(getMasterProcedureExecutor(), htd, splitKeys);
+ splitAndTruncate(families, splitKeys, tableName, regions);
+ }
+
+ private void splitAndTruncate(final String[] families, final byte[][] splitKeys,
+ TableName tableName, HRegionInfo[] regions) throws IOException, InterruptedException {
// load enough data so the table can split
- MasterProcedureTestingUtility.loadData(
- UTIL.getConnection(), tableName, 5000, splitKeys, families);
+ MasterProcedureTestingUtility.loadData(UTIL.getConnection(), tableName, 5000, splitKeys,
+ families);
assertEquals(5000, UTIL.countRows(tableName));
UTIL.getHBaseAdmin().split(tableName);
UTIL.waitUntilAllRegionsAssigned(tableName);
- //wait until split really happens
- while(UTIL.getHBaseAdmin().getTableRegions(tableName).size() <= regions.length) {
+ // wait until split really happens
+ while (UTIL.getHBaseAdmin().getTableRegions(tableName).size() <= regions.length) {
Thread.sleep(50);
}
// disable the table
@@ -265,10 +284,9 @@ public class TestTruncateTableProcedure {
// truncate the table
final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
long procId = ProcedureTestingUtility.submitAndWait(procExec,
- new TruncateTableProcedure(procExec.getEnvironment(), tableName, true));
+ new TruncateTableProcedure(procExec.getEnvironment(), tableName, true));
ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
UTIL.waitUntilAllRegionsAssigned(tableName);
-
}
}
[5/5] hbase git commit: HBASE-19391 Calling
HRegion#initializeRegionInternals from a region replica can still re-create a
region directory; ADDEDNDUM to fix TestRegionOpen failure
Posted by ap...@apache.org.
HBASE-19391 Calling HRegion#initializeRegionInternals from a region replica can still re-create a region directory; ADDEDNDUM to fix TestRegionOpen failure
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/569f16ea
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/569f16ea
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/569f16ea
Branch: refs/heads/branch-1.4
Commit: 569f16ea2c477ae5abee6d242a8806b815457f32
Parents: 2d12ae2
Author: Michael Stack <st...@apache.org>
Authored: Thu Feb 22 09:14:35 2018 -0800
Committer: Andrew Purtell <ap...@apache.org>
Committed: Fri Feb 23 17:03:03 2018 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/regionserver/TestRegionOpen.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/569f16ea/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionOpen.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionOpen.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionOpen.java
index edb1d52..62aabce 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionOpen.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionOpen.java
@@ -47,6 +47,7 @@ import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
import org.junit.AfterClass;
import org.junit.BeforeClass;
+import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -97,7 +98,8 @@ public class TestRegionOpen {
assertEquals(2, exec.getCompletedTaskCount());
}
- @Test(timeout = 60000)
+ @Ignore // Needs rewrite since HBASE-19391 which returns null out of createRegionOnFileSystem
+ @Test
public void testNonExistentRegionReplica() throws Exception {
final TableName tableName = TableName.valueOf(name.getMethodName());
final byte[] FAMILYNAME = Bytes.toBytes("fam");
[2/5] hbase git commit: HBASE-19553,
Old replica regions should be cleared from AM memory after primary
region split or merge
Posted by ap...@apache.org.
HBASE-19553, Old replica regions should be cleared from AM memory after primary region split or merge
Signed-off-by: Ted Yu <te...@apache.org>
Signed-off-by: Huaxiang Sun <hu...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/29b03a50
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/29b03a50
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/29b03a50
Branch: refs/heads/branch-1.4
Commit: 29b03a5023de55d96768f53f471e09aa97598750
Parents: 569f16e
Author: Pankaj Kumar <pa...@huawei.com>
Authored: Thu Jan 25 13:25:22 2018 +0800
Committer: Andrew Purtell <ap...@apache.org>
Committed: Fri Feb 23 17:03:03 2018 -0800
----------------------------------------------------------------------
.../hadoop/hbase/master/AssignmentManager.java | 17 ++++
.../apache/hadoop/hbase/client/TestAdmin1.java | 102 +++++++++++++++++--
2 files changed, 111 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/29b03a50/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index b8c088a..c7339ef 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -4394,6 +4394,8 @@ public class AssignmentManager extends ZooKeeperListener {
LOG.warn("Couldn't assign all replica(s) of region " + mergedHri+ " because of " +
ie.getMessage());
}
+ // Remove merged region's replica from AM's memory
+ clearReplicaRegions(c);
}
private void doSplittingOfReplicas(final HRegionInfo parentHri, final HRegionInfo hri_a,
@@ -4437,6 +4439,21 @@ public class AssignmentManager extends ZooKeeperListener {
} catch (InterruptedException e) {
LOG.warn("Caught exception " + e + " while trying to assign replica(s) of daughter(s)");
}
+ // Remove parent region's replica from AM's memory
+ clearReplicaRegions(c);
+ }
+
+ /*
+ * Clear the replica regions after region split or merge.
+ */
+ private void clearReplicaRegions(Collection<List<HRegionInfo>> regionInfos) {
+ for (List<HRegionInfo> regionInfoList : regionInfos) {
+ for (HRegionInfo regionInfo : regionInfoList) {
+ if (!RegionReplicaUtil.isDefaultReplica(regionInfo)) {
+ regionStates.deleteRegion(regionInfo);
+ }
+ }
+ }
}
private void prepareDaughterReplicaForAssignment(HRegionInfo daughterHri, HRegionInfo parentHri,
http://git-wip-us.apache.org/repos/asf/hbase/blob/29b03a50/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java
index 279b304..5885024 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java
@@ -50,6 +50,7 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotDisabledException;
import org.apache.hadoop.hbase.TableNotEnabledException;
import org.apache.hadoop.hbase.TableNotFoundException;
+import org.apache.hadoop.hbase.Waiter;
import org.apache.hadoop.hbase.executor.EventHandler;
import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
import org.apache.hadoop.hbase.util.Bytes;
@@ -57,7 +58,9 @@ import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.zookeeper.ZKTableStateClientSideReader;
import org.apache.hadoop.hbase.ZooKeeperConnectionException;
import org.apache.hadoop.hbase.exceptions.MergeRegionException;
+import org.apache.hadoop.hbase.master.AssignmentManager;
import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.master.RegionState;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.RequestConverter;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService;
@@ -1146,15 +1149,8 @@ public class TestAdmin1 {
// regions.
// Set up a table with 3 regions and replication set to 3
TableName tableName = TableName.valueOf("testSplitAndMergeWithReplicaTable");
- HTableDescriptor desc = new HTableDescriptor(tableName);
- desc.setRegionReplication(3);
byte[] cf = "f".getBytes();
- HColumnDescriptor hcd = new HColumnDescriptor(cf);
- desc.addFamily(hcd);
- byte[][] splitRows = new byte[2][];
- splitRows[0] = new byte[]{(byte)'4'};
- splitRows[1] = new byte[]{(byte)'7'};
- TEST_UTIL.getHBaseAdmin().createTable(desc, splitRows);
+ createReplicaTable(tableName, cf);
List<HRegion> oldRegions;
do {
oldRegions = TEST_UTIL.getHBaseCluster().getRegions(tableName);
@@ -1243,6 +1239,96 @@ public class TestAdmin1 {
}
/**
+ * Test case to validate whether parent's replica regions are cleared from AM's memory after
+ * SPLIT/MERGE.
+ */
+ @Test
+ public void testRegionStateCleanupFromAMMemoryAfterRegionSplitAndMerge() throws Exception {
+ final TableName tableName =
+ TableName.valueOf("testRegionStateCleanupFromAMMemoryAfterRegionSplitAndMerge");
+ createReplicaTable(tableName, "f".getBytes());
+ final int regionReplication = admin.getTableDescriptor(tableName).getRegionReplication();
+
+ List<Pair<HRegionInfo, ServerName>> regions = MetaTableAccessor.getTableRegionsAndLocations(
+ TEST_UTIL.getZooKeeperWatcher(), TEST_UTIL.getConnection(), tableName);
+ assertEquals(9, regions.size());
+ final int primaryRegionCount = regions.size() / regionReplication;
+
+ final AssignmentManager am = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager();
+ List<HRegionInfo> splitRegions =
+ am.getRegionStates().getRegionByStateOfTable(tableName).get(RegionState.State.SPLIT);
+ assertEquals(0, splitRegions.size());
+
+ // Validate region split
+ byte[] regionName = regions.get(0).getFirst().getRegionName();
+ try {
+ TEST_UTIL.getHBaseAdmin().split(regionName, Bytes.toBytes('2'));
+ } catch (IllegalArgumentException ex) {
+ fail("Exception occured during region split" + ex);
+ }
+
+ // Wait for replica region to become online
+ TEST_UTIL.waitFor(60000, 500, new Waiter.Predicate<IOException>() {
+ @Override
+ public boolean evaluate() throws IOException {
+ return am.getRegionStates().getRegionByStateOfTable(tableName).get(RegionState.State.OPEN)
+ .size() == (primaryRegionCount + 1) * regionReplication;
+ }
+ });
+
+ regions = MetaTableAccessor.getTableRegionsAndLocations(TEST_UTIL.getZooKeeperWatcher(),
+ TEST_UTIL.getConnection(), tableName);
+ assertEquals(12, regions.size());
+ final int primaryRegionCountAfterSplit = regions.size() / regionReplication;
+
+ // Split region after region split
+ splitRegions =
+ am.getRegionStates().getRegionByStateOfTable(tableName).get(RegionState.State.SPLIT);
+ // Parent's replica region should be removed from AM's memory.
+ assertEquals(1, splitRegions.size());
+
+ // Validate region merge
+ HRegionInfo regionA = regions.get(3).getFirst();
+ HRegionInfo regionB = regions.get(6).getFirst();
+ try {
+ TEST_UTIL.getHBaseAdmin().mergeRegions(regionA.getRegionName(), regionB.getRegionName(),
+ true);
+ } catch (IllegalArgumentException ex) {
+ fail("Exception occured during region merge" + ex);
+ }
+
+ // Wait for replica regions to become online
+ TEST_UTIL.waitFor(60000, 500, new Waiter.Predicate<IOException>() {
+ @Override
+ public boolean evaluate() throws IOException {
+ return am.getRegionStates().getRegionByStateOfTable(tableName).get(RegionState.State.OPEN)
+ .size() == (primaryRegionCountAfterSplit - 1) * regionReplication;
+ }
+ });
+
+ regions = MetaTableAccessor.getTableRegionsAndLocations(TEST_UTIL.getZooKeeperWatcher(),
+ TEST_UTIL.getConnection(), tableName);
+ assertEquals(9, regions.size());
+ // Offline region after region merge
+ List<HRegionInfo> offlineRegions =
+ am.getRegionStates().getRegionByStateOfTable(tableName).get(RegionState.State.OFFLINE);
+ // Parent's replica region should be removed from AM's memory.
+ assertEquals(0, offlineRegions.size());
+ }
+
+ private byte[] createReplicaTable(TableName tableName, byte[] cf) throws IOException {
+ HTableDescriptor desc = new HTableDescriptor(tableName);
+ desc.setRegionReplication(3);
+ HColumnDescriptor hcd = new HColumnDescriptor(cf);
+ desc.addFamily(hcd);
+ byte[][] splitRows = new byte[2][];
+ splitRows[0] = new byte[] { (byte) '4' };
+ splitRows[1] = new byte[] { (byte) '7' };
+ TEST_UTIL.getHBaseAdmin().createTable(desc, splitRows);
+ return cf;
+ }
+
+ /**
* HADOOP-2156
* @throws IOException
*/
[3/5] hbase git commit: HBASE-20052
TestRegionOpen#testNonExistentRegionReplica fails due to NPE
Posted by ap...@apache.org.
HBASE-20052 TestRegionOpen#testNonExistentRegionReplica fails due to NPE
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/1b0b8d95
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/1b0b8d95
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/1b0b8d95
Branch: refs/heads/branch-1.4
Commit: 1b0b8d95364fe9b2fb03b0a74b4c6bfba8af02d2
Parents: 29b03a5
Author: tedyu <yu...@gmail.com>
Authored: Thu Feb 22 12:41:13 2018 -0800
Committer: Andrew Purtell <ap...@apache.org>
Committed: Fri Feb 23 17:03:03 2018 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java | 3 +--
.../java/org/apache/hadoop/hbase/regionserver/TestRegionOpen.java | 2 --
2 files changed, 1 insertion(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/1b0b8d95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
index 1a66063..bc3b039 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
@@ -967,12 +967,11 @@ public class HRegionFileSystem {
// Write HRI to a file in case we need to recover hbase:meta
regionFs.writeRegionInfoOnFilesystem(false);
- return regionFs;
} else {
if (LOG.isDebugEnabled())
LOG.debug("Skipping creation of .regioninfo file for " + regionInfo);
}
- return null;
+ return regionFs;
}
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/1b0b8d95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionOpen.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionOpen.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionOpen.java
index 62aabce..f45df18 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionOpen.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionOpen.java
@@ -47,7 +47,6 @@ import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
import org.junit.AfterClass;
import org.junit.BeforeClass;
-import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -98,7 +97,6 @@ public class TestRegionOpen {
assertEquals(2, exec.getCompletedTaskCount());
}
- @Ignore // Needs rewrite since HBASE-19391 which returns null out of createRegionOnFileSystem
@Test
public void testNonExistentRegionReplica() throws Exception {
final TableName tableName = TableName.valueOf(name.getMethodName());