You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by jx...@apache.org on 2014/03/25 20:34:55 UTC
svn commit: r1581479 [4/9] - in /hbase/trunk:
hbase-client/src/main/java/org/apache/hadoop/hbase/catalog/
hbase-client/src/main/java/org/apache/hadoop/hbase/client/
hbase-it/src/test/java/org/apache/hadoop/hbase/
hbase-it/src/test/java/org/apache/hadoo...
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java?rev=1581479&r1=1581478&r2=1581479&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java Tue Mar 25 19:34:52 2014
@@ -150,6 +150,7 @@ public class StochasticLoadBalancer exte
costFunctions = new CostFunction[]{
new RegionCountSkewCostFunction(conf),
+ new RegionOnMasterCostFunction(conf),
new MoveCostFunction(conf),
localityCost,
new TableSkewCostFunction(conf),
@@ -191,14 +192,18 @@ public class StochasticLoadBalancer exte
*/
@Override
public List<RegionPlan> balanceCluster(Map<ServerName, List<HRegionInfo>> clusterState) {
- if (!needsBalance(new ClusterLoadState(clusterState))) {
+ List<RegionPlan> plans = balanceMasterRegions(clusterState);
+ if (plans != null) {
+ return plans;
+ }
+ if (!needsBalance(new ClusterLoadState(masterServerName, clusterState))) {
return null;
}
long startTime = EnvironmentEdgeManager.currentTimeMillis();
// Keep track of servers to iterate through them.
- Cluster cluster = new Cluster(clusterState, loads, regionFinder);
+ Cluster cluster = new Cluster(masterServerName, clusterState, loads, regionFinder);
double currentCost = computeCost(cluster, Double.MAX_VALUE);
double initCost = currentCost;
@@ -257,7 +262,7 @@ public class StochasticLoadBalancer exte
metricsBalancer.balanceCluster(endTime - startTime);
if (initCost > currentCost) {
- List<RegionPlan> plans = createRegionPlans(cluster);
+ plans = createRegionPlans(cluster);
if (LOG.isDebugEnabled()) {
LOG.debug("Finished computing new load balance plan. Computation took "
+ (endTime - startTime) + "ms to try " + step
@@ -563,8 +568,8 @@ public class StochasticLoadBalancer exte
*/
public abstract static class CostFunction {
- private float multiplier = 0;
- private Configuration conf;
+ protected float multiplier = 0;
+ protected Configuration conf;
CostFunction(Configuration c) {
this.conf = c;
@@ -742,6 +747,29 @@ public class StochasticLoadBalancer exte
}
}
+ /**
+ * Compute the cost of a potential cluster configuration based upon if putting
+ * user regions on the master regionserver.
+ */
+ public static class RegionOnMasterCostFunction extends CostFunction {
+
+ private static final String REGION_ON_MASTER_COST_KEY =
+ "hbase.master.balancer.stochastic.regionOnMasterCost";
+ private static final float DEFAULT_REGION_ON_MASTER__COST = 1000;
+
+ RegionOnMasterCostFunction(Configuration conf) {
+ super(conf);
+ this.setMultiplier(conf.getFloat(
+ REGION_ON_MASTER_COST_KEY, DEFAULT_REGION_ON_MASTER__COST));
+ }
+
+ @Override
+ double cost(Cluster cluster) {
+ double max = cluster.numRegions;
+ double value = cluster.numUserRegionsOnMaster;
+ return scale(0, max, value);
+ }
+ }
/**
* Compute a cost of a potential cluster configuration based upon where
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java?rev=1581479&r1=1581478&r2=1581479&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java Tue Mar 25 19:34:52 2014
@@ -148,7 +148,7 @@ public class CreateTableHandler extends
LOG.info("Create table " + tableName);
try {
- MasterCoprocessorHost cpHost = ((HMaster) this.server).getCoprocessorHost();
+ MasterCoprocessorHost cpHost = ((HMaster) this.server).getMasterCoprocessorHost();
if (cpHost != null) {
cpHost.preCreateTableHandler(this.hTableDescriptor, this.newRegions);
}
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java?rev=1581479&r1=1581478&r2=1581479&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java Tue Mar 25 19:34:52 2014
@@ -61,7 +61,7 @@ public class DeleteTableHandler extends
protected void handleTableOperation(List<HRegionInfo> regions)
throws IOException, KeeperException {
MasterCoprocessorHost cpHost = ((HMaster) this.server)
- .getCoprocessorHost();
+ .getMasterCoprocessorHost();
if (cpHost != null) {
cpHost.preDeleteTableHandler(this.tableName);
}
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/DisableTableHandler.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/DisableTableHandler.java?rev=1581479&r1=1581478&r2=1581479&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/DisableTableHandler.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/DisableTableHandler.java Tue Mar 25 19:34:52 2014
@@ -129,7 +129,7 @@ public class DisableTableHandler extends
try {
LOG.info("Attempting to disable table " + this.tableName);
MasterCoprocessorHost cpHost = ((HMaster) this.server)
- .getCoprocessorHost();
+ .getMasterCoprocessorHost();
if (cpHost != null) {
cpHost.preDisableTableHandler(this.tableName);
}
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java?rev=1581479&r1=1581478&r2=1581479&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java Tue Mar 25 19:34:52 2014
@@ -22,7 +22,6 @@ import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.concurrent.ExecutorService;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -48,7 +47,6 @@ import org.apache.hadoop.hbase.master.Ta
import org.apache.hadoop.hbase.master.TableLockManager.TableLock;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.zookeeper.KeeperException;
-import org.htrace.Trace;
/**
* Handler to run enable of a table.
@@ -139,7 +137,7 @@ public class EnableTableHandler extends
try {
LOG.info("Attempting to enable the table " + this.tableName);
MasterCoprocessorHost cpHost = ((HMaster) this.server)
- .getCoprocessorHost();
+ .getMasterCoprocessorHost();
if (cpHost != null) {
cpHost.preEnableTableHandler(this.tableName);
}
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ModifyTableHandler.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ModifyTableHandler.java?rev=1581479&r1=1581478&r2=1581479&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ModifyTableHandler.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ModifyTableHandler.java Tue Mar 25 19:34:52 2014
@@ -60,7 +60,7 @@ public class ModifyTableHandler extends
@Override
protected void handleTableOperation(List<HRegionInfo> hris)
throws IOException {
- MasterCoprocessorHost cpHost = ((HMaster) this.server).getCoprocessorHost();
+ MasterCoprocessorHost cpHost = ((HMaster) this.server).getMasterCoprocessorHost();
if (cpHost != null) {
cpHost.preModifyTableHandler(this.tableName, this.htd);
}
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TableAddFamilyHandler.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TableAddFamilyHandler.java?rev=1581479&r1=1581478&r2=1581479&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TableAddFamilyHandler.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TableAddFamilyHandler.java Tue Mar 25 19:34:52 2014
@@ -61,7 +61,7 @@ public class TableAddFamilyHandler exten
protected void handleTableOperation(List<HRegionInfo> hris)
throws IOException {
MasterCoprocessorHost cpHost = ((HMaster) this.server)
- .getCoprocessorHost();
+ .getMasterCoprocessorHost();
if(cpHost != null){
cpHost.preAddColumnHandler(this.tableName, this.familyDesc);
}
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TableDeleteFamilyHandler.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TableDeleteFamilyHandler.java?rev=1581479&r1=1581478&r2=1581479&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TableDeleteFamilyHandler.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TableDeleteFamilyHandler.java Tue Mar 25 19:34:52 2014
@@ -57,7 +57,7 @@ public class TableDeleteFamilyHandler ex
@Override
protected void handleTableOperation(List<HRegionInfo> hris) throws IOException {
MasterCoprocessorHost cpHost = ((HMaster) this.server)
- .getCoprocessorHost();
+ .getMasterCoprocessorHost();
if (cpHost != null) {
cpHost.preDeleteColumnHandler(this.tableName, this.familyName);
}
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TableModifyFamilyHandler.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TableModifyFamilyHandler.java?rev=1581479&r1=1581478&r2=1581479&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TableModifyFamilyHandler.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TableModifyFamilyHandler.java Tue Mar 25 19:34:52 2014
@@ -56,7 +56,7 @@ public class TableModifyFamilyHandler ex
@Override
protected void handleTableOperation(List<HRegionInfo> regions) throws IOException {
MasterCoprocessorHost cpHost = ((HMaster) this.server)
- .getCoprocessorHost();
+ .getMasterCoprocessorHost();
if (cpHost != null) {
cpHost.preModifyColumnHandler(this.tableName, this.familyDesc);
}
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java?rev=1581479&r1=1581478&r2=1581479&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java Tue Mar 25 19:34:52 2014
@@ -137,7 +137,6 @@ public class SnapshotManager extends Mas
private boolean stopped;
private MasterServices master; // Needed by TableEventHandlers
- private MetricsMaster metricsMaster;
private ProcedureCoordinator coordinator;
// Is snapshot feature enabled?
@@ -172,7 +171,6 @@ public class SnapshotManager extends Mas
ProcedureCoordinator coordinator, ExecutorService pool)
throws IOException, UnsupportedOperationException {
this.master = master;
- this.metricsMaster = metricsMaster;
this.rootDir = master.getMasterFileSystem().getRootDir();
checkSnapshotSupport(master.getConfiguration(), master.getMasterFileSystem());
@@ -260,7 +258,7 @@ public class SnapshotManager extends Mas
public void deleteSnapshot(SnapshotDescription snapshot) throws SnapshotDoesNotExistException, IOException {
// call coproc pre hook
- MasterCoprocessorHost cpHost = master.getCoprocessorHost();
+ MasterCoprocessorHost cpHost = master.getMasterCoprocessorHost();
if (cpHost != null) {
cpHost.preDeleteSnapshot(snapshot);
}
@@ -547,7 +545,7 @@ public class SnapshotManager extends Mas
.build();
// call pre coproc hook
- MasterCoprocessorHost cpHost = master.getCoprocessorHost();
+ MasterCoprocessorHost cpHost = master.getMasterCoprocessorHost();
if (cpHost != null) {
cpHost.preSnapshot(snapshot, desc);
}
@@ -668,7 +666,7 @@ public class SnapshotManager extends Mas
public void restoreSnapshot(SnapshotDescription reqSnapshot) throws IOException {
FileSystem fs = master.getMasterFileSystem().getFileSystem();
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(reqSnapshot, rootDir);
- MasterCoprocessorHost cpHost = master.getCoprocessorHost();
+ MasterCoprocessorHost cpHost = master.getMasterCoprocessorHost();
// check if the snapshot exists
if (!fs.exists(snapshotDir)) {
@@ -1005,7 +1003,6 @@ public class SnapshotManager extends Mas
public void initialize(MasterServices master, MetricsMaster metricsMaster) throws KeeperException,
IOException, UnsupportedOperationException {
this.master = master;
- this.metricsMaster = metricsMaster;
this.rootDir = master.getMasterFileSystem().getRootDir();
checkSnapshotSupport(master.getConfiguration(), master.getMasterFileSystem());
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/monitoring/StateDumpServlet.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/monitoring/StateDumpServlet.java?rev=1581479&r1=1581478&r2=1581479&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/monitoring/StateDumpServlet.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/monitoring/StateDumpServlet.java Tue Mar 25 19:34:52 2014
@@ -26,6 +26,7 @@ import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.executor.ExecutorService;
import org.apache.hadoop.hbase.executor.ExecutorService.ExecutorStatus;
import org.apache.hadoop.hbase.util.VersionInfo;
@@ -45,6 +46,10 @@ public abstract class StateDumpServlet e
" on " + org.apache.hadoop.util.VersionInfo.getDate());
}
+ protected boolean isShowQueueDump(Configuration conf){
+ return conf.getBoolean("hbase.regionserver.servlet.show.queuedump", true);
+ }
+
protected long getTailKbParam(HttpServletRequest request) {
String param = request.getParameter("tailkb");
if (param == null) {
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AnnotationReadingPriorityFunction.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AnnotationReadingPriorityFunction.java?rev=1581479&r1=1581478&r2=1581479&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AnnotationReadingPriorityFunction.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AnnotationReadingPriorityFunction.java Tue Mar 25 19:34:52 2014
@@ -24,7 +24,6 @@ import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.NotServingRegionException;
import org.apache.hadoop.hbase.ipc.PriorityFunction;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionRequest;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CompactRegionRequest;
@@ -38,7 +37,7 @@ import org.apache.hadoop.hbase.protobuf.
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanRequest;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier;
import org.apache.hadoop.hbase.protobuf.generated.RPCProtos.RequestHeader;
-import org.apache.hadoop.hbase.regionserver.HRegionServer.QosPriority;
+import org.apache.hadoop.hbase.regionserver.RSRpcServices.QosPriority;
import com.google.common.annotations.VisibleForTesting;
import com.google.protobuf.Message;
@@ -72,7 +71,7 @@ class AnnotationReadingPriorityFunction
private final Map<String, Integer> annotatedQos;
//We need to mock the regionserver instance for some unit tests (set via
//setRegionServer method.
- private HRegionServer hRegionServer;
+ private RSRpcServices rpcServices;
@SuppressWarnings("unchecked")
private final Class<? extends Message>[] knownArgumentClasses = new Class[]{
GetRegionInfoRequest.class,
@@ -92,10 +91,9 @@ class AnnotationReadingPriorityFunction
private final Map<String, Map<Class<? extends Message>, Method>> methodMap =
new HashMap<String, Map<Class<? extends Message>, Method>>();
- AnnotationReadingPriorityFunction(final HRegionServer hrs) {
- this.hRegionServer = hrs;
+ AnnotationReadingPriorityFunction(final RSRpcServices rpcServices) {
Map<String, Integer> qosMap = new HashMap<String, Integer>();
- for (Method m : HRegionServer.class.getMethods()) {
+ for (Method m : RSRpcServices.class.getMethods()) {
QosPriority p = m.getAnnotation(QosPriority.class);
if (p != null) {
// Since we protobuf'd, and then subsequently, when we went with pb style, method names
@@ -107,6 +105,7 @@ class AnnotationReadingPriorityFunction
qosMap.put(capitalizedMethodName, p.priority());
}
}
+ this.rpcServices = rpcServices;
this.annotatedQos = qosMap;
if (methodMap.get("getRegion") == null) {
methodMap.put("hasRegion", new HashMap<Class<? extends Message>, Method>());
@@ -129,16 +128,6 @@ class AnnotationReadingPriorityFunction
return strBuilder.toString();
}
- public boolean isMetaRegion(byte[] regionName) {
- HRegion region;
- try {
- region = hRegionServer.getRegion(regionName);
- } catch (NotServingRegionException ignored) {
- return false;
- }
- return region.getRegionInfo().isMetaTable();
- }
-
@Override
public int getPriority(RequestHeader header, Message param) {
String methodName = header.getMethodName();
@@ -167,7 +156,7 @@ class AnnotationReadingPriorityFunction
if (hasRegion != null && (Boolean)hasRegion.invoke(param, (Object[])null)) {
Method getRegion = methodMap.get("getRegion").get(rpcArgClass);
regionSpecifier = (RegionSpecifier)getRegion.invoke(param, (Object[])null);
- HRegion region = hRegionServer.getRegion(regionSpecifier);
+ HRegion region = rpcServices.getRegion(regionSpecifier);
if (region.getRegionInfo().isMetaTable()) {
if (LOG.isTraceEnabled()) {
LOG.trace("High priority because region=" + region.getRegionNameAsString());
@@ -187,7 +176,7 @@ class AnnotationReadingPriorityFunction
if (!request.hasScannerId()) {
return HConstants.NORMAL_QOS;
}
- RegionScanner scanner = hRegionServer.getScanner(request.getScannerId());
+ RegionScanner scanner = rpcServices.getScanner(request.getScannerId());
if (scanner != null && scanner.getRegionInfo().isMetaRegion()) {
if (LOG.isTraceEnabled()) {
// Scanner requests are small in size so TextFormat version should not overwhelm log.
@@ -201,6 +190,6 @@ class AnnotationReadingPriorityFunction
@VisibleForTesting
void setRegionServer(final HRegionServer hrs) {
- this.hRegionServer = hrs;
+ this.rpcServices = hrs.getRSRpcServices();
}
}