You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2010/08/26 21:14:46 UTC
svn commit: r989883 - in /hbase/branches/0.90_master_rewrite/src:
main/java/org/apache/hadoop/hbase/master/
main/java/org/apache/hadoop/hbase/regionserver/
test/java/org/apache/hadoop/hbase/ test/java/org/apache/hadoop/hbase/avro/
test/java/org/apache/...
Author: stack
Date: Thu Aug 26 19:14:45 2010
New Revision: 989883
URL: http://svn.apache.org/viewvc?rev=989883&view=rev
Log:
Made the balancer test pass (About 3 or 4 small bugs in balancer)
All tests but an avro, a rest and a WAL test are passing now.
Starting merge from TRUNK
Modified:
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/TestFullLogReconstruction.java
hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java
hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/avro/TestAvroServer.java
hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java
Modified: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java?rev=989883&r1=989882&r2=989883&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java (original)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java Thu Aug 26 19:14:45 2010
@@ -1034,7 +1034,9 @@ public class AssignmentManager extends Z
}
/**
- * @return A clone of current assignments
+ * @return A clone of current assignments. Note, this is assignments only.
+ * If a new server has come in and it has no regions, it will not be included
+ * in the returned Map.
*/
Map<HServerInfo, List<HRegionInfo>> getAssignments() {
// This is an EXPENSIVE clone. Cloning though is the safest thing to do.
@@ -1046,7 +1048,11 @@ public class AssignmentManager extends Z
result = new HashMap<HServerInfo, List<HRegionInfo>>(this.servers.size());
for (Map.Entry<HServerInfo, List<HRegionInfo>> e: this.servers.entrySet()) {
List<HRegionInfo> shallowCopy = new ArrayList<HRegionInfo>(e.getValue());
- result.put(e.getKey(), shallowCopy);
+ HServerInfo clone = new HServerInfo(e.getKey());
+ // Set into server load the number of regions this server is carrying
+ // The load balancer calculation needs it at least and its handy.
+ clone.getLoad().setNumberOfRegions(e.getValue().size());
+ result.put(clone, shallowCopy);
}
}
return result;
Modified: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/HMaster.java?rev=989883&r1=989882&r2=989883&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/HMaster.java (original)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/HMaster.java Thu Aug 26 19:14:45 2010
@@ -26,6 +26,7 @@ import java.lang.management.RuntimeMXBea
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.net.UnknownHostException;
+import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicReference;
@@ -537,6 +538,14 @@ implements HMasterInterface, HMasterRegi
}
Map<HServerInfo, List<HRegionInfo>> assignments =
this.assignmentManager.getAssignments();
+ // Returned Map from AM does not include mention of servers w/o assignments.
+ for (Map.Entry<String, HServerInfo> e:
+ this.serverManager.getOnlineServers().entrySet()) {
+ HServerInfo hsi = e.getValue();
+ if (!assignments.containsKey(hsi)) {
+ assignments.put(hsi, new ArrayList<HRegionInfo>());
+ }
+ }
List<RegionPlan> plans = this.balancer.balanceCluster(assignments);
if (plans == null || plans.isEmpty()) return;
for (RegionPlan plan: plans) {
Modified: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java?rev=989883&r1=989882&r2=989883&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java (original)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java Thu Aug 26 19:14:45 2010
@@ -141,6 +141,7 @@ public class LoadBalancer {
// Iterate so we can count regions as we build the map
for(Map.Entry<HServerInfo, List<HRegionInfo>> server :
clusterState.entrySet()) {
+ server.getKey().getLoad().setNumberOfRegions(server.getValue().size());
numRegions += server.getKey().getLoad().getNumberOfRegions();
serversByLoad.put(server.getKey(), server.getValue());
}
@@ -181,11 +182,16 @@ public class LoadBalancer {
serversOverloaded++;
List<HRegionInfo> regions = server.getValue();
int numToOffload = Math.min(regionCount - max, regions.size());
- for(int i=0; i<numToOffload; i++) {
- regionsToMove.add(new RegionPlan(regions.get(i), serverInfo, null));
+ int numTaken = 0;
+ for (HRegionInfo hri: regions) {
+ // Don't rebalance meta regions.
+ if (hri.isMetaRegion()) continue;
+ regionsToMove.add(new RegionPlan(hri, serverInfo, null));
+ numTaken++;
+ if (numTaken >= numToOffload) break;
}
serverBalanceInfo.put(serverInfo,
- new BalanceInfo(numToOffload, (-1)*numToOffload));
+ new BalanceInfo(numToOffload, (-1)*numTaken));
}
// Walk down least loaded, filling each to the min
@@ -235,6 +241,7 @@ public class LoadBalancer {
int idx =
balanceInfo == null ? 0 : balanceInfo.getNextRegionForUnload();
HRegionInfo region = server.getValue().get(idx);
+ if (region.isMetaRegion()) continue; // Don't move meta regions.
regionsToMove.add(new RegionPlan(region, server.getKey(), null));
if(--neededRegions == 0) {
// No more regions needed, done shedding
@@ -559,11 +566,11 @@ public class LoadBalancer {
}
/**
- * Get the region name for the region this plan is for.
- * @return region name
+ * Get the encoded region name for the region this plan is for.
+ * @return Encoded region name
*/
public String getRegionName() {
- return this.hri.getRegionNameAsString();
+ return this.hri.getEncodedName();
}
public HRegionInfo getRegionInfo() {
Modified: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java?rev=989883&r1=989882&r2=989883&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java (original)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java Thu Aug 26 19:14:45 2010
@@ -383,9 +383,10 @@ public class ServerManager {
}
/**
- * @return Read-only map of servers to serverinfo.
+ * @return Read-only map of servers to serverinfo
*/
public Map<String, HServerInfo> getOnlineServers() {
+ // Presumption is that iterating the returned Map is OK.
synchronized (this.onlineServers) {
return Collections.unmodifiableMap(this.onlineServers);
}
Modified: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java?rev=989883&r1=989882&r2=989883&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java (original)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java Thu Aug 26 19:14:45 2010
@@ -288,7 +288,7 @@ public class StoreFile {
*/
public long getMaxSequenceId() {
if (this.sequenceid == -1) {
- throw new IllegalAccessError("Has not been initialized");
+ throw new IllegalAccessError(this + " has not been initialized");
}
return this.sequenceid;
}
Modified: hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/TestFullLogReconstruction.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/TestFullLogReconstruction.java?rev=989883&r1=989882&r2=989883&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/TestFullLogReconstruction.java (original)
+++ hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/TestFullLogReconstruction.java Thu Aug 26 19:14:45 2010
@@ -20,16 +20,19 @@
package org.apache.hadoop.hbase;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import static org.junit.Assert.assertEquals;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.util.Bytes;
-import org.junit.*;
-import static org.junit.Assert.assertEquals;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
public class TestFullLogReconstruction {
Modified: hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java?rev=989883&r1=989882&r2=989883&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java (original)
+++ hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java Thu Aug 26 19:14:45 2010
@@ -33,6 +33,7 @@ import org.apache.hadoop.hbase.regionser
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.JVMClusterUtil;
+import org.apache.hadoop.hbase.util.Threads;
/**
* Test whether region rebalancing works. (HBASE-71)
@@ -104,17 +105,23 @@ public class TestRegionRebalancing exten
LOG.debug("Adding 2nd region server.");
// add a region server - total of 2
cluster.startRegionServer();
+ while (cluster.getMaster().getServerManager().getOnlineServers().size() < 2) {
+ Threads.sleep(100);
+ }
+ cluster.getMaster().balance();
assertRegionsAreBalanced();
// add a region server - total of 3
LOG.debug("Adding 3rd region server.");
cluster.startRegionServer();
+ cluster.getMaster().balance();
assertRegionsAreBalanced();
// kill a region server - total of 2
LOG.debug("Killing the 3rd region server.");
cluster.stopRegionServer(2, false);
cluster.waitOnRegionServer(2);
+ cluster.getMaster().balance();
assertRegionsAreBalanced();
// start two more region servers - total of 4
@@ -122,12 +129,14 @@ public class TestRegionRebalancing exten
cluster.startRegionServer();
LOG.debug("Adding 4th region server");
cluster.startRegionServer();
+ cluster.getMaster().balance();
assertRegionsAreBalanced();
for (int i = 0; i < 6; i++){
LOG.debug("Adding " + (i + 5) + "th region server");
cluster.startRegionServer();
}
+ cluster.getMaster().balance();
assertRegionsAreBalanced();
}
@@ -166,7 +175,7 @@ public class TestRegionRebalancing exten
for (HRegionServer server : servers) {
int serverLoad = server.getOnlineRegions().size();
- LOG.debug(server.hashCode() + " Avg: " + avg + " actual: " + serverLoad);
+ LOG.debug(server.getServerName() + " Avg: " + avg + " actual: " + serverLoad);
if (!(avg > 2.0 && serverLoad <= avgLoadPlusSlop
&& serverLoad >= avgLoadMinusSlop)) {
LOG.debug(server.hashCode() + " Isn't balanced!!! Avg: " + avg +
Modified: hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/avro/TestAvroServer.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/avro/TestAvroServer.java?rev=989883&r1=989882&r2=989883&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/avro/TestAvroServer.java (original)
+++ hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/avro/TestAvroServer.java Thu Aug 26 19:14:45 2010
@@ -22,30 +22,23 @@ import static org.junit.Assert.assertFal
import static org.junit.Assert.assertTrue;
import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.hadoop.hbase.HBaseClusterTestCase;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Ignore;
-import org.junit.Test;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericArray;
import org.apache.avro.generic.GenericData;
-
+import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.avro.generated.AColumn;
import org.apache.hadoop.hbase.avro.generated.AColumnValue;
import org.apache.hadoop.hbase.avro.generated.AFamilyDescriptor;
import org.apache.hadoop.hbase.avro.generated.AGet;
import org.apache.hadoop.hbase.avro.generated.APut;
-import org.apache.hadoop.hbase.avro.generated.AResult;
import org.apache.hadoop.hbase.avro.generated.ATableDescriptor;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
/**
* Unit testing for AvroServer.HBaseImpl, a part of the
@@ -129,7 +122,7 @@ public class TestAvroServer {
tableA.maxFileSize = 123456L;
impl.modifyTable(tableAname, tableA);
- assertEquals((long) impl.describeTable(tableAname).maxFileSize, 123456L);
+ assertEquals(123456L, (long) impl.describeTable(tableAname).maxFileSize);
impl.enableTable(tableAname);
assertTrue(impl.isTableEnabled(tableAname));
Modified: hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java?rev=989883&r1=989882&r2=989883&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java (original)
+++ hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java Thu Aug 26 19:14:45 2010
@@ -77,7 +77,7 @@ public class TestRemoteTable extends HBa
htd.addFamily(new HColumnDescriptor(COLUMN_2));
htd.addFamily(new HColumnDescriptor(COLUMN_3));
admin.createTable(htd);
- HTable table = new HTable(TABLE);
+ HTable table = new HTable(conf, TABLE);
Put put = new Put(ROW_1);
put.add(COLUMN_1, QUALIFIER_1, TS_2, VALUE_1);
table.put(put);