You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ns...@apache.org on 2011/10/11 19:43:56 UTC
svn commit: r1181952 - in /hbase/branches/0.89/src:
main/java/org/apache/hadoop/hbase/master/ test/java/org/apache/hadoop/hbase/
Author: nspiegelberg
Date: Tue Oct 11 17:43:56 2011
New Revision: 1181952
URL: http://svn.apache.org/viewvc?rev=1181952&view=rev
Log:
Enable preferred assignment in MiniHBaseCluster for unit tests
Summary:
Previously, there were some unit tests failure caused by preferred assignment.
So this preferred assignment was disabled for MiniHBaseCluster since the diff
https://phabricator.fb.com/D271660 .
It is always a good idea to keep the MiniHBaseCluster as the same behavior as a
real HBase cluster does.
Also I have fully understood why the unit test would fail before.
There are 2 reasons contributing to this problem.
1) When enabling the preferred assignment, the master will hold the region
assignment for its best locality region server for a period of time (1 mins), if
its best locality region server has not checked in master.
But there is a potential bug in the code that it will also hold the region
assignment EVEN there is no best locality region server for this region.
This bug only happens when the HBase client tries to create a new table when
the master is trying to hold regions assignment for its best locality region
server, which caused the client throws RetryExhausted Exception.
So fix the bug in this diff.
2) There are some unit tests which depends on load balancer running.
So let these unit test function wait for the load balancer running.
To sum up, this diff is going to enable to preferred assignment in
MiniHBaseCluster for all unit tests, fix 1 assignment bug and update 2 unit
tests.
Test Plan:
Have run all the unit tests.
Only 1 failed unit test case is failed, but this unit test has been failed
recently. Pretty sure that it is not caused by this diff.
testCloseHRegionRetry(org.apache.hadoop.hbase.regionserver.TestHRegionCloseRetry)
Also I will test them on the dev cluster later.
Reviewers: kannan, kranganathan, pritam, gqchen
Reviewed By: kannan
CC: , hbase@lists, kannan
Differential Revision: 312480
Modified:
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/master/RegionManager.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/TestMultiParallelPut.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/master/RegionManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/master/RegionManager.java?rev=1181952&r1=1181951&r2=1181952&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/master/RegionManager.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/master/RegionManager.java Tue Oct 11 17:43:56 2011
@@ -607,12 +607,14 @@ public class RegionManager {
String preferredHost =
this.master.getPreferredRegionToRegionServerMapping().get(name);
- if (preferredHost != null && hostName.startsWith(preferredHost)) {
- LOG.debug("Doing Preferred Region Assignment for : " + name +
- " to the " + hostName);
- } else if (holdRegionForBestRegionserver ||
- quickStartRegionServerSet.contains(preferredHost)) {
- continue;
+ if (preferredHost != null) {
+ if (hostName.startsWith(preferredHost)) {
+ LOG.debug("Doing Preferred Region Assignment for : " + name +
+ " to the " + hostName);
+ } else if (holdRegionForBestRegionserver ||
+ quickStartRegionServerSet.contains(preferredHost)) {
+ continue;
+ }
}
}
Modified: hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java?rev=1181952&r1=1181951&r2=1181952&view=diff
==============================================================================
--- hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java (original)
+++ hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java Tue Oct 11 17:43:56 2011
@@ -58,6 +58,8 @@ public class MiniHBaseCluster {
UGI = UserGroupInformation.getCurrentUGI();
}
+ static long PREFERRED_ASSIGNMENT = 1000L;
+ static long WAIT_FOR_LOADBALANCER = 2000L;
/**
* Start a MiniHBaseCluster.
* @param conf Configuration to be used for cluster
@@ -68,7 +70,10 @@ public class MiniHBaseCluster {
throws IOException {
this.conf = conf;
conf.set(HConstants.MASTER_PORT, "0");
- conf.setLong("hbase.master.applyPreferredAssignment.period", 0);
+ conf.setLong("hbase.master.applyPreferredAssignment.period",
+ PREFERRED_ASSIGNMENT);
+ conf.setLong("hbase.master.holdRegionForBestLocality.period",
+ PREFERRED_ASSIGNMENT / 5);
init(numRegionServers);
}
Modified: hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/TestMultiParallelPut.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/TestMultiParallelPut.java?rev=1181952&r1=1181951&r2=1181952&view=diff
==============================================================================
--- hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/TestMultiParallelPut.java (original)
+++ hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/TestMultiParallelPut.java Tue Oct 11 17:43:56 2011
@@ -102,6 +102,9 @@ public class TestMultiParallelPut extend
r.getValue(BYTES_FAMILY, QUALIFIER)));
}
+ // waiting for load balancer running
+ Thread.sleep(MiniHBaseCluster.WAIT_FOR_LOADBALANCER);
+
HBaseAdmin admin = new HBaseAdmin(conf);
ClusterStatus cs = admin.getClusterStatus();
int expectedServerCount = 2;
Modified: hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java?rev=1181952&r1=1181951&r2=1181952&view=diff
==============================================================================
--- hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java (original)
+++ hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java Tue Oct 11 17:43:56 2011
@@ -144,6 +144,13 @@ public class TestRegionRebalancing exten
float slop = conf.getFloat("hbase.regions.slop", (float)0.1);
if (slop <= 0) slop = 1;
+ // waiting for load balancer running
+ try {
+ Thread.sleep(MiniHBaseCluster.WAIT_FOR_LOADBALANCER);
+ } catch (InterruptedException e1) {
+ LOG.error("Got InterruptedException when waiting for load balance " + e1);
+ }
+
for (int i = 0; i < 5; i++) {
success = true;
// make sure all the regions are reassigned before we test balance