You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2012/05/03 08:20:28 UTC
svn commit: r1333319 [1/4] - in /hbase/trunk/src:
main/jamon/org/apache/hadoop/hbase/tmpl/master/
main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/
main/java/org/apache/hadoop/hbase/ main/java/org/apache/hadoop/hbase/ipc/
main/java/org/apache/hadoo...
Author: stack
Date: Thu May 3 06:20:26 2012
New Revision: 1333319
URL: http://svn.apache.org/viewvc?rev=1333319&view=rev
Log:
HBASE-5444 Add PB-based calls to HMasterRegionInterface
Added:
hbase/trunk/src/main/java/org/apache/hadoop/hbase/ServerLoad.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/ipc/RegionServerStatusProtocol.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/protobuf/generated/RegionServerStatusProtos.java
hbase/trunk/src/main/protobuf/RegionServerStatus.proto
Removed:
hbase/trunk/src/main/java/org/apache/hadoop/hbase/ipc/HMasterRegionInterface.java
Modified:
hbase/trunk/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
hbase/trunk/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon
hbase/trunk/src/main/java/org/apache/hadoop/hbase/ClusterStatus.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRpcMetrics.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/ipc/Invocation.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/MXBean.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/MXBeanImpl.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/MasterDumpServlet.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/protobuf/generated/HBaseProtos.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
hbase/trunk/src/main/protobuf/hbase.proto
hbase/trunk/src/main/resources/hbase-webapps/master/table.jsp
hbase/trunk/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestMXBean.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestServerCustomProtocol.java
Modified: hbase/trunk/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon?rev=1333319&r1=1333318&r2=1333319&view=diff
==============================================================================
--- hbase/trunk/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon (original)
+++ hbase/trunk/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon Thu May 3 06:20:26 2012
@@ -37,7 +37,7 @@ org.apache.hadoop.hbase.util.JvmVersion;
org.apache.hadoop.hbase.util.FSUtils;
org.apache.hadoop.hbase.master.HMaster;
org.apache.hadoop.hbase.HConstants;
-org.apache.hadoop.hbase.HServerLoad;
+org.apache.hadoop.hbase.ServerLoad;
org.apache.hadoop.hbase.ServerName;
org.apache.hadoop.hbase.client.HBaseAdmin;
org.apache.hadoop.hbase.client.HConnectionManager;
@@ -213,11 +213,12 @@ org.apache.hadoop.hbase.HBaseConfigurati
// or be set to 0 to get ephemeral ports
int infoPort = master.getConfiguration().getInt("hbase.regionserver.info.port", 60030);
String url = "http://" + serverName.getHostname() + ":" + infoPort + "/";
- HServerLoad hsl = master.getServerManager().getLoad(serverName);
- String loadStr = hsl == null? "-": hsl.toString();
- if (hsl != null) {
- totalRegions += hsl.getNumberOfRegions();
- totalRequests += hsl.getNumberOfRequests();
+ ServerLoad sl = master.getServerManager().getLoad(serverName);
+ String loadStr = sl == null? "-": sl.toString();
+ if (sl != null) {
+ totalRegions += sl.getRegionLoadsCount();
+ // Is this correct? Adding a rate to a measure.
+ totalRequests += sl.getRequestsPerSecond();
}
long startcode = serverName.getStartcode();
</%java>
Modified: hbase/trunk/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon?rev=1333319&r1=1333318&r2=1333319&view=diff
==============================================================================
--- hbase/trunk/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon (original)
+++ hbase/trunk/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon Thu May 3 06:20:26 2012
@@ -32,7 +32,8 @@ org.apache.hadoop.hbase.regionserver.met
org.apache.hadoop.hbase.util.Bytes;
org.apache.hadoop.hbase.HConstants;
org.apache.hadoop.hbase.HServerInfo;
-org.apache.hadoop.hbase.HServerLoad;
+org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerLoad;
+org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionLoad;
org.apache.hadoop.hbase.HRegionInfo;
org.apache.hadoop.hbase.ServerName;
org.apache.hadoop.hbase.HBaseConfiguration;
@@ -118,7 +119,7 @@ String url = "http://" + host + "/";
</%java>
<%for HRegionInfo r: onlineRegions %>
<%java>
- HServerLoad.RegionLoad load = regionServer.createRegionLoad(r.getEncodedName());
+ RegionLoad load = regionServer.createRegionLoad(r.getEncodedName());
</%java>
<tr><td><% r.getRegionNameAsString() %></td>
<td><% Bytes.toStringBinary(r.getStartKey()) %></td><td><% Bytes.toStringBinary(r.getEndKey()) %></td>
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/ClusterStatus.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/ClusterStatus.java?rev=1333319&r1=1333318&r2=1333319&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/ClusterStatus.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/ClusterStatus.java Thu May 3 06:20:26 2012
@@ -29,10 +29,15 @@ import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
+import java.util.Set;
import java.util.TreeMap;
-
+import java.util.HashSet;
+import org.apache.hadoop.hbase.HServerLoad;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
+import org.apache.hadoop.hbase.ServerLoad;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hbase.HServerLoad.RegionLoad;
import org.apache.hadoop.hbase.master.AssignmentManager.RegionState;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.VersionMismatchException;
@@ -89,14 +94,48 @@ public class ClusterStatus extends Versi
}
public ClusterStatus(final String hbaseVersion, final String clusterid,
- final Map<ServerName, HServerLoad> servers,
+ final Map<ServerName, ServerLoad> servers,
final Collection<ServerName> deadServers,
final ServerName master,
final Collection<ServerName> backupMasters,
final Map<String, RegionState> rit,
final String[] masterCoprocessors) {
this.hbaseVersion = hbaseVersion;
- this.liveServers = servers;
+
+ // TODO: This conversion of ServerLoad to HServerLoad is temporary,
+ // will be cleaned up in HBASE-5445. Using the ClusterStatus proto brings
+ // in a lot of other changes, so it makes sense to break this up.
+ Map<ServerName, HServerLoad> convertedLoad =
+ new HashMap<ServerName,HServerLoad>();
+ for (Map.Entry<ServerName,ServerLoad> entry : servers.entrySet()) {
+ ServerLoad sl = entry.getValue();
+
+ Map<byte[],RegionLoad> regionLoad = new HashMap<byte[],RegionLoad>();
+ for (HBaseProtos.RegionLoad rl : sl.getRegionLoadsList()) {
+ Set<String> regionCoprocessors = new HashSet<String>();
+ for (HBaseProtos.Coprocessor coprocessor
+ : rl.getCoprocessorsList()) {
+ regionCoprocessors.add(coprocessor.getName());
+ }
+
+ byte [] regionName = rl.getRegionSpecifier().getValue().toByteArray();
+ RegionLoad converted = new RegionLoad(regionName,
+ rl.getStores(),rl.getStorefiles(),rl.getStoreUncompressedSizeMB(),
+ rl.getStorefileSizeMB(),rl.getMemstoreSizeMB(),
+ rl.getStorefileIndexSizeMB(),rl.getRootIndexSizeKB(),
+ rl.getTotalStaticIndexSizeKB(),rl.getTotalStaticBloomSizeKB(),
+ rl.getReadRequestsCount(),rl.getWriteRequestsCount(),
+ rl.getTotalCompactingKVs(),rl.getCurrentCompactedKVs(),
+ regionCoprocessors);
+ regionLoad.put(regionName, converted);
+ }
+
+ HServerLoad hsl = new HServerLoad(sl.getTotalNumberOfRequests(),
+ sl.getRequestsPerSecond(),sl.getUsedHeapMB(),sl.getMaxHeapMB(),
+ regionLoad,new HashSet<String>(Arrays.asList(masterCoprocessors)));
+ convertedLoad.put(entry.getKey(), hsl);
+ }
+ this.liveServers = convertedLoad;
this.deadServers = deadServers;
this.master = master;
this.backupMasters = backupMasters;
Added: hbase/trunk/src/main/java/org/apache/hadoop/hbase/ServerLoad.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/ServerLoad.java?rev=1333319&view=auto
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/ServerLoad.java (added)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/ServerLoad.java Thu May 3 06:20:26 2012
@@ -0,0 +1,155 @@
+/**
+ * Copyright The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase;
+
+import java.util.List;
+import java.util.TreeSet;
+
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionLoad;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.Coprocessor;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * This class is used for exporting current state of load on a RegionServer.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class ServerLoad {
+ public ServerLoad(HBaseProtos.ServerLoad serverLoad) {
+ this.serverLoad = serverLoad;
+ }
+
+ /* @return the underlying ServerLoad protobuf object */
+ public HBaseProtos.ServerLoad getServerLoadPB() {
+ return serverLoad;
+ }
+
+ protected HBaseProtos.ServerLoad serverLoad;
+
+ /* @return number of requests per second since last report. */
+ public int getRequestsPerSecond() {
+ return serverLoad.getRequestsPerSecond();
+ }
+ public boolean hasRequestsPerSecond() {
+ return serverLoad.hasRequestsPerSecond();
+ }
+
+ /* @return total Number of requests from the start of the region server. */
+ public int getTotalNumberOfRequests() {
+ return serverLoad.getTotalNumberOfRequests();
+ }
+ public boolean hasTotalNumberOfRequests() {
+ return serverLoad.hasTotalNumberOfRequests();
+ }
+
+ /* @return the amount of used heap, in MB. */
+ public int getUsedHeapMB() {
+ return serverLoad.getUsedHeapMB();
+ }
+ public boolean hasUsedHeapMB() {
+ return serverLoad.hasUsedHeapMB();
+ }
+
+ /* @return the maximum allowable size of the heap, in MB. */
+ public int getMaxHeapMB() {
+ return serverLoad.getMaxHeapMB();
+ }
+ public boolean hasMaxHeapMB() {
+ return serverLoad.hasMaxHeapMB();
+ }
+
+ /* Returns list of RegionLoads, which contain information on the load of individual regions. */
+ public List<RegionLoad> getRegionLoadsList() {
+ return serverLoad.getRegionLoadsList();
+ }
+ public RegionLoad getRegionLoads(int index) {
+ return serverLoad.getRegionLoads(index);
+ }
+ public int getRegionLoadsCount() {
+ return serverLoad.getRegionLoadsCount();
+ }
+
+ /**
+ * @return the list Regionserver-level coprocessors, e.g., WALObserver implementations.
+ * Region-level coprocessors, on the other hand, are stored inside the RegionLoad objects.
+ */
+ public List<Coprocessor> getCoprocessorsList() {
+ return serverLoad.getCoprocessorsList();
+ }
+ public Coprocessor getCoprocessors(int index) {
+ return serverLoad.getCoprocessors(index);
+ }
+ public int getCoprocessorsCount() {
+ return serverLoad.getCoprocessorsCount();
+ }
+
+ /**
+ * Return the RegionServer-level coprocessors from a ServerLoad pb.
+ * @param sl - ServerLoad
+ * @return string array of loaded RegionServer-level coprocessors
+ */
+ public static String[] getRegionServerCoprocessors(ServerLoad sl) {
+ if (sl == null) {
+ return null;
+ }
+
+ List<Coprocessor> list = sl.getCoprocessorsList();
+ String [] ret = new String[list.size()];
+ int i = 0;
+ for (Coprocessor elem : list) {
+ ret[i++] = elem.getName();
+ }
+
+ return ret;
+ }
+
+ /**
+ * Return the RegionServer-level and Region-level coprocessors
+ * from a ServerLoad pb.
+ * @param sl - ServerLoad
+ * @return string array of loaded RegionServer-level and
+ * Region-level coprocessors
+ */
+ public static String[] getAllCoprocessors(ServerLoad sl) {
+ if (sl == null) {
+ return null;
+ }
+
+ // Need a set to remove duplicates, but since generated Coprocessor class
+ // is not Comparable, make it a Set<String> instead of Set<Coprocessor>
+ TreeSet<String> coprocessSet = new TreeSet<String>();
+ for (Coprocessor coprocessor : sl.getCoprocessorsList()) {
+ coprocessSet.add(coprocessor.getName());
+ }
+ for (RegionLoad rl : sl.getRegionLoadsList()) {
+ for (Coprocessor coprocessor : rl.getCoprocessorsList()) {
+ coprocessSet.add(coprocessor.getName());
+ }
+ }
+
+ return coprocessSet.toArray(new String[0]);
+ }
+
+ public static final ServerLoad EMPTY_SERVERLOAD =
+ new ServerLoad(HBaseProtos.ServerLoad.newBuilder().build());
+}
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRpcMetrics.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRpcMetrics.java?rev=1333319&r1=1333318&r2=1333319&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRpcMetrics.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRpcMetrics.java Thu May 3 06:20:26 2012
@@ -29,6 +29,7 @@ import org.apache.hadoop.metrics.Metrics
import org.apache.hadoop.metrics.MetricsUtil;
import org.apache.hadoop.metrics.Updater;
import org.apache.hadoop.metrics.util.*;
+import org.apache.hadoop.hbase.ipc.RegionServerStatusProtocol;
import java.lang.reflect.Method;
@@ -64,7 +65,7 @@ public class HBaseRpcMetrics implements
context.registerUpdater(this);
initMethods(HMasterInterface.class);
- initMethods(HMasterRegionInterface.class);
+ initMethods(RegionServerStatusProtocol.class);
initMethods(HRegionInterface.class);
rpcStatistics = new HBaseRPCStatistics(this.registry, hostName, port);
}
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java?rev=1333319&r1=1333318&r2=1333319&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java Thu May 3 06:20:26 2012
@@ -43,6 +43,7 @@ import java.nio.channels.SocketChannel;
import java.nio.channels.WritableByteChannel;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
@@ -133,6 +134,10 @@ public abstract class HBaseServer implem
new ThreadLocal<RpcServer>();
private volatile boolean started = false;
+ // For generated protocol classes which doesn't have VERSION field
+ private static final Map<Class<?>, Long>
+ PROTOCOL_VERSION = new HashMap<Class<?>, Long>();
+
private static final Map<String, Class<? extends VersionedProtocol>>
PROTOCOL_CACHE =
new ConcurrentHashMap<String, Class<? extends VersionedProtocol>>();
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/ipc/Invocation.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/ipc/Invocation.java?rev=1333319&r1=1333318&r2=1333319&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/ipc/Invocation.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/ipc/Invocation.java Thu May 3 06:20:26 2012
@@ -37,6 +37,8 @@ import org.apache.hadoop.hbase.client.Cl
import org.apache.hadoop.hbase.io.HbaseObjectWritable;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ClientService;
+import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionServerStatusService;
+import org.apache.hadoop.hbase.ipc.RegionServerStatusProtocol;
import org.apache.hadoop.io.VersionMismatchException;
import org.apache.hadoop.io.VersionedWritable;
@@ -61,6 +63,8 @@ public class Invocation extends Versione
Long.valueOf(ClientProtocol.VERSION));
PROTOCOL_VERSION.put(AdminService.BlockingInterface.class,
Long.valueOf(AdminProtocol.VERSION));
+ PROTOCOL_VERSION.put(RegionServerStatusService.BlockingInterface.class,
+ Long.valueOf(RegionServerStatusProtocol.VERSION));
}
// For protobuf protocols, which use ServiceException, instead of IOException
@@ -70,6 +74,7 @@ public class Invocation extends Versione
static {
PROTOBUF_PROTOCOLS.add(ClientProtocol.class);
PROTOBUF_PROTOCOLS.add(AdminProtocol.class);
+ PROTOBUF_PROTOCOLS.add(RegionServerStatusProtocol.class);
}
private static byte RPC_VERSION = 1;
Added: hbase/trunk/src/main/java/org/apache/hadoop/hbase/ipc/RegionServerStatusProtocol.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/ipc/RegionServerStatusProtocol.java?rev=1333319&view=auto
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/ipc/RegionServerStatusProtocol.java (added)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/ipc/RegionServerStatusProtocol.java Thu May 3 06:20:26 2012
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.ipc;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hbase.ipc.VersionedProtocol;
+import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionServerStatusService;
+import org.apache.hadoop.hbase.security.TokenInfo;
+import org.apache.hadoop.security.KerberosInfo;
+
+/**
+ * Protocol that a RegionServer uses to communicate its status to the Master.
+ */
+@KerberosInfo(
+ serverPrincipal = "hbase.master.kerberos.principal")
+@TokenInfo("HBASE_AUTH_TOKEN")
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public interface RegionServerStatusProtocol extends
+ RegionServerStatusService.BlockingInterface, VersionedProtocol {
+ public static final long VERSION = 1L;
+}
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java?rev=1333319&r1=1333318&r2=1333319&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java Thu May 3 06:20:26 2012
@@ -76,6 +76,7 @@ import org.apache.hadoop.hbase.master.me
import org.apache.hadoop.hbase.regionserver.RegionAlreadyInTransitionException;
import org.apache.hadoop.hbase.regionserver.RegionOpeningState;
import org.apache.hadoop.hbase.regionserver.RegionServerStoppedException;
+import org.apache.hadoop.hbase.ServerLoad;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.util.Threads;
@@ -3171,10 +3172,10 @@ public class AssignmentManager extends Z
}
}
}
- Map<ServerName, HServerLoad> onlineSvrs = this.serverManager.getOnlineServers();
+ Map<ServerName, ServerLoad> onlineSvrs = this.serverManager.getOnlineServers();
// Take care of servers w/o assignments.
for (Map<ServerName,List<HRegionInfo>> map : result.values()) {
- for (Map.Entry<ServerName, HServerLoad> svrEntry: onlineSvrs.entrySet()) {
+ for (Map.Entry<ServerName, ServerLoad> svrEntry: onlineSvrs.entrySet()) {
if (!map.containsKey(svrEntry.getKey())) {
map.put(svrEntry.getKey(), new ArrayList<HRegionInfo>());
}
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java?rev=1333319&r1=1333318&r2=1333319&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java Thu May 3 06:20:26 2012
@@ -77,7 +77,8 @@ import org.apache.hadoop.hbase.executor.
import org.apache.hadoop.hbase.ipc.HBaseRPC;
import org.apache.hadoop.hbase.ipc.HBaseServer;
import org.apache.hadoop.hbase.ipc.HMasterInterface;
-import org.apache.hadoop.hbase.ipc.HMasterRegionInterface;
+import org.apache.hadoop.hbase.ipc.RegionServerStatusProtocol;
+import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.ipc.ProtocolSignature;
import org.apache.hadoop.hbase.ipc.RpcServer;
import org.apache.hadoop.hbase.master.handler.CreateTableHandler;
@@ -112,12 +113,21 @@ import org.apache.hadoop.hbase.zookeeper
import org.apache.hadoop.hbase.zookeeper.RegionServerTracker;
import org.apache.hadoop.hbase.zookeeper.ZKUtil;
import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
-import org.apache.hadoop.io.MapWritable;
-import org.apache.hadoop.io.Text;
import org.apache.hadoop.metrics.util.MBeanUtil;
import org.apache.hadoop.net.DNS;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.Watcher;
+import org.apache.hadoop.hbase.ServerLoad;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameStringPair;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
+import com.google.protobuf.RpcController;
+import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionServerReportRequest;
+import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionServerReportResponse;
+import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.ReportRSFatalErrorRequest;
+import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.ReportRSFatalErrorResponse;
+import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionServerStartupRequest;
+import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionServerStartupResponse;
+import com.google.protobuf.ServiceException;
/**
* HMaster is the "master server" for HBase. An HBase cluster has one active
@@ -133,12 +143,12 @@ import org.apache.zookeeper.Watcher;
* <p>You can also shutdown just this master. Call {@link #stopMaster()}.
*
* @see HMasterInterface
- * @see HMasterRegionInterface
+ * @see MasterRegionInterface
* @see Watcher
*/
@InterfaceAudience.Private
public class HMaster extends HasThread
-implements HMasterInterface, HMasterRegionInterface, MasterServices,
+implements HMasterInterface, RegionServerStatusProtocol, MasterServices,
Server {
private static final Log LOG = LogFactory.getLog(HMaster.class.getName());
@@ -262,7 +272,7 @@ Server {
int numHandlers = conf.getInt("hbase.master.handler.count",
conf.getInt("hbase.regionserver.handler.count", 25));
this.rpcServer = HBaseRPC.getServer(this,
- new Class<?>[]{HMasterInterface.class, HMasterRegionInterface.class},
+ new Class<?>[]{HMasterInterface.class, RegionServerStatusProtocol.class},
initialIsa.getHostName(), // BindAddress is IP we got for this server.
initialIsa.getPort(),
numHandlers,
@@ -564,7 +574,7 @@ Server {
// Not registered; add it.
LOG.info("Registering server found up in zk but who has not yet " +
"reported in: " + sn);
- this.serverManager.recordNewServer(sn, HServerLoad.EMPTY_HSERVERLOAD);
+ this.serverManager.recordNewServer(sn, ServerLoad.EMPTY_SERVERLOAD);
}
}
@@ -795,8 +805,8 @@ Server {
throws IOException {
if (HMasterInterface.class.getName().equals(protocol)) {
return new ProtocolSignature(HMasterInterface.VERSION, null);
- } else if (HMasterRegionInterface.class.getName().equals(protocol)) {
- return new ProtocolSignature(HMasterRegionInterface.VERSION, null);
+ } else if (RegionServerStatusProtocol.class.getName().equals(protocol)) {
+ return new ProtocolSignature(RegionServerStatusProtocol.VERSION, null);
}
throw new IOException("Unknown protocol: " + protocol);
}
@@ -804,8 +814,8 @@ Server {
public long getProtocolVersion(String protocol, long clientVersion) {
if (HMasterInterface.class.getName().equals(protocol)) {
return HMasterInterface.VERSION;
- } else if (HMasterRegionInterface.class.getName().equals(protocol)) {
- return HMasterRegionInterface.VERSION;
+ } else if (RegionServerStatusProtocol.class.getName().equals(protocol)) {
+ return RegionServerStatusProtocol.VERSION;
}
// unknown protocol
LOG.warn("Version requested for unimplemented protocol: "+protocol);
@@ -952,18 +962,25 @@ Server {
}
@Override
- public MapWritable regionServerStartup(final int port,
- final long serverStartCode, final long serverCurrentTime)
- throws IOException {
+ public RegionServerStartupResponse regionServerStartup(
+ RpcController controller, RegionServerStartupRequest request) throws ServiceException {
// Register with server manager
- InetAddress ia = getRemoteInetAddress(port, serverStartCode);
- ServerName rs = this.serverManager.regionServerStartup(ia, port,
- serverStartCode, serverCurrentTime);
- // Send back some config info
- MapWritable mw = createConfigurationSubset();
- mw.put(new Text(HConstants.KEY_FOR_HOSTNAME_SEEN_BY_MASTER),
- new Text(rs.getHostname()));
- return mw;
+ try {
+ InetAddress ia = getRemoteInetAddress(request.getPort(), request.getServerStartCode());
+ ServerName rs = this.serverManager.regionServerStartup(ia, request.getPort(),
+ request.getServerStartCode(), request.getServerCurrentTime());
+
+ // Send back some config info
+ RegionServerStartupResponse.Builder resp = createConfigurationSubset();
+ NameStringPair.Builder entry = NameStringPair.newBuilder()
+ .setName(HConstants.KEY_FOR_HOSTNAME_SEEN_BY_MASTER)
+ .setValue(rs.getHostname());
+ resp.addMapEntries(entry.build());
+
+ return resp.build();
+ } catch(IOException ioe) {
+ throw new ServiceException(ioe);
+ }
}
/**
@@ -981,32 +998,49 @@ Server {
* @return Subset of configuration to pass initializing regionservers: e.g.
* the filesystem to use and root directory to use.
*/
- protected MapWritable createConfigurationSubset() {
- MapWritable mw = addConfig(new MapWritable(), HConstants.HBASE_DIR);
- return addConfig(mw, "fs.default.name");
+ protected RegionServerStartupResponse.Builder createConfigurationSubset() {
+ RegionServerStartupResponse.Builder resp = addConfig(
+ RegionServerStartupResponse.newBuilder(), HConstants.HBASE_DIR);
+ return addConfig(resp, "fs.default.name");
}
- private MapWritable addConfig(final MapWritable mw, final String key) {
- mw.put(new Text(key), new Text(this.conf.get(key)));
- return mw;
+ private RegionServerStartupResponse.Builder addConfig(
+ final RegionServerStartupResponse.Builder resp, final String key) {
+ NameStringPair.Builder entry = NameStringPair.newBuilder()
+ .setName(key)
+ .setValue(this.conf.get(key));
+ resp.addMapEntries(entry.build());
+ return resp;
}
@Override
- public void regionServerReport(final byte [] sn, final HServerLoad hsl)
- throws IOException {
- this.serverManager.regionServerReport(ServerName.parseVersionedServerName(sn), hsl);
- if (hsl != null && this.metrics != null) {
- // Up our metrics.
- this.metrics.incrementRequests(hsl.getTotalNumberOfRequests());
+ public RegionServerReportResponse regionServerReport(
+ RpcController controller,RegionServerReportRequest request) throws ServiceException {
+ try {
+ HBaseProtos.ServerLoad sl = request.getLoad();
+ this.serverManager.regionServerReport(ProtobufUtil.toServerName(request.getServer()), new ServerLoad(sl));
+ if (sl != null && this.metrics != null) {
+ // Up our metrics.
+ this.metrics.incrementRequests(sl.getTotalNumberOfRequests());
+ }
+ } catch(IOException ioe) {
+ throw new ServiceException(ioe);
}
+
+ return RegionServerReportResponse.newBuilder().build();
}
@Override
- public void reportRSFatalError(byte [] sn, String errorText) {
- String msg = "Region server " + Bytes.toString(sn) +
+ public ReportRSFatalErrorResponse reportRSFatalError(
+ RpcController controller, ReportRSFatalErrorRequest request) throws ServiceException {
+ String errorText = request.getErrorMessage();
+ ServerName sn = ProtobufUtil.toServerName(request.getServer());
+ String msg = "Region server " + Bytes.toString(sn.getVersionedBytes()) +
" reported a fatal error:\n" + errorText;
LOG.error(msg);
rsFatals.add(msg);
+
+ return ReportRSFatalErrorResponse.newBuilder().build();
}
public boolean isMasterRunning() {
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/MXBean.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/MXBean.java?rev=1333319&r1=1333318&r2=1333319&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/MXBean.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/MXBean.java Thu May 3 06:20:26 2012
@@ -20,7 +20,7 @@ package org.apache.hadoop.hbase.master;
import java.util.Map;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
-import org.apache.hadoop.hbase.HServerLoad;
+import org.apache.hadoop.hbase.ServerLoad;
/**
* This is the JMX management interface for Hbase master information
@@ -101,7 +101,7 @@ public interface MXBean {
* Get the live region servers
* @return Live region servers
*/
- public Map<String, HServerLoad> getRegionServers();
+ public Map<String, ServerLoad> getRegionServers();
/**
* Get the dead region servers
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/MXBeanImpl.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/MXBeanImpl.java?rev=1333319&r1=1333318&r2=1333319&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/MXBeanImpl.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/MXBeanImpl.java Thu May 3 06:20:26 2012
@@ -23,7 +23,7 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
-import org.apache.hadoop.hbase.HServerLoad;
+import org.apache.hadoop.hbase.ServerLoad;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.master.AssignmentManager.RegionState;
@@ -77,9 +77,9 @@ public class MXBeanImpl implements MXBea
}
@Override
- public Map<String, HServerLoad> getRegionServers() {
- Map<String, HServerLoad> data = new HashMap<String, HServerLoad>();
- for (final Entry<ServerName, HServerLoad> entry :
+ public Map<String, ServerLoad> getRegionServers() {
+ Map<String, ServerLoad> data = new HashMap<String, ServerLoad>();
+ for (final Entry<ServerName, ServerLoad> entry :
master.getServerManager().getOnlineServers().entrySet()) {
data.put(entry.getKey().getServerName(),
entry.getValue());
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/MasterDumpServlet.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/MasterDumpServlet.java?rev=1333319&r1=1333318&r2=1333319&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/MasterDumpServlet.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/MasterDumpServlet.java Thu May 3 06:20:26 2012
@@ -32,8 +32,8 @@ import javax.servlet.http.HttpServletRes
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HServerInfo;
-import org.apache.hadoop.hbase.HServerLoad;
import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.ServerLoad;
import org.apache.hadoop.hbase.master.AssignmentManager.RegionState;
import org.apache.hadoop.hbase.monitoring.LogMonitoring;
import org.apache.hadoop.hbase.monitoring.StateDumpServlet;
@@ -114,9 +114,9 @@ public class MasterDumpServlet extends S
}
private void dumpServers(HMaster master, PrintWriter out) {
- Map<ServerName, HServerLoad> servers =
+ Map<ServerName, ServerLoad> servers =
master.getServerManager().getOnlineServers();
- for (Map.Entry<ServerName, HServerLoad> e : servers.entrySet()) {
+ for (Map.Entry<ServerName, ServerLoad> e : servers.entrySet()) {
out.println(e.getKey() + ": " + e.getValue());
}
}
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java?rev=1333319&r1=1333318&r2=1333319&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java Thu May 3 06:20:26 2012
@@ -38,7 +38,9 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.hbase.ClockOutOfSyncException;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HServerAddress;
-import org.apache.hadoop.hbase.HServerLoad;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.ServerLoad;
import org.apache.hadoop.hbase.PleaseHoldException;
import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.ServerName;
@@ -51,7 +53,6 @@ import org.apache.hadoop.hbase.client.Re
import org.apache.hadoop.hbase.master.handler.MetaServerShutdownHandler;
import org.apache.hadoop.hbase.master.handler.ServerShutdownHandler;
import org.apache.hadoop.hbase.monitoring.MonitoredTask;
-import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.regionserver.RegionOpeningState;
/**
@@ -75,8 +76,8 @@ public class ServerManager {
private volatile boolean clusterShutdown = false;
/** Map of registered servers to their current load */
- private final Map<ServerName, HServerLoad> onlineServers =
- new ConcurrentHashMap<ServerName, HServerLoad>();
+ private final Map<ServerName, ServerLoad> onlineServers =
+ new ConcurrentHashMap<ServerName, ServerLoad>();
// TODO: This is strange to have two maps but HSI above is used on both sides
/**
@@ -154,11 +155,11 @@ public class ServerManager {
checkClockSkew(sn, serverCurrentTime);
checkIsDead(sn, "STARTUP");
checkAlreadySameHostPort(sn);
- recordNewServer(sn, HServerLoad.EMPTY_HSERVERLOAD);
+ recordNewServer(sn, ServerLoad.EMPTY_SERVERLOAD);
return sn;
}
- void regionServerReport(ServerName sn, HServerLoad hsl)
+ void regionServerReport(ServerName sn, ServerLoad sl)
throws YouAreDeadException, PleaseHoldException {
checkIsDead(sn, "REPORT");
if (!this.onlineServers.containsKey(sn)) {
@@ -169,9 +170,9 @@ public class ServerManager {
// The only thing we are skipping is passing back to the regionserver
// the ServerName to use. Here we presume a master has already done
// that so we'll press on with whatever it gave us for ServerName.
- recordNewServer(sn, hsl);
+ recordNewServer(sn, sl);
} else {
- this.onlineServers.put(sn, hsl);
+ this.onlineServers.put(sn, sl);
}
}
@@ -255,9 +256,9 @@ public class ServerManager {
* @param hsl
* @param serverName The remote servers name.
*/
- void recordNewServer(final ServerName serverName, final HServerLoad hsl) {
+ void recordNewServer(final ServerName serverName, final ServerLoad sl) {
LOG.info("Registering server=" + serverName);
- this.onlineServers.put(serverName, hsl);
+ this.onlineServers.put(serverName, sl);
this.serverConnections.remove(serverName);
}
@@ -265,7 +266,7 @@ public class ServerManager {
* @param serverName
* @return HServerLoad if serverName is known else null
*/
- public HServerLoad getLoad(final ServerName serverName) {
+ public ServerLoad getLoad(final ServerName serverName) {
return this.onlineServers.get(serverName);
}
@@ -274,7 +275,7 @@ public class ServerManager {
* @return HServerLoad if serverName is known else null
* @deprecated Use {@link #getLoad(HServerAddress)}
*/
- public HServerLoad getLoad(final HServerAddress address) {
+ public ServerLoad getLoad(final HServerAddress address) {
ServerName sn = new ServerName(address.toString(), ServerName.NON_STARTCODE);
ServerName actual =
ServerName.findServerWithSameHostnamePort(this.getOnlineServersList(), sn);
@@ -291,9 +292,9 @@ public class ServerManager {
int totalLoad = 0;
int numServers = 0;
double averageLoad = 0.0;
- for (HServerLoad hsl: this.onlineServers.values()) {
+ for (ServerLoad sl: this.onlineServers.values()) {
numServers++;
- totalLoad += hsl.getNumberOfRegions();
+ totalLoad += sl.getRegionLoadsCount();
}
averageLoad = (double)totalLoad / (double)numServers;
return averageLoad;
@@ -308,7 +309,7 @@ public class ServerManager {
/**
* @return Read-only map of servers to serverinfo
*/
- public Map<ServerName, HServerLoad> getOnlineServers() {
+ public Map<ServerName, ServerLoad> getOnlineServers() {
// Presumption is that iterating the returned Map is OK.
synchronized (this.onlineServers) {
return Collections.unmodifiableMap(this.onlineServers);
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java?rev=1333319&r1=1333318&r2=1333319&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java Thu May 3 06:20:26 2012
@@ -101,6 +101,7 @@ import org.apache.hadoop.hbase.protobuf.
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameBytesPair;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameStringPair;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionLoad;
import org.apache.hadoop.hbase.regionserver.RegionOpeningState;
import org.apache.hadoop.hbase.regionserver.wal.HLog;
import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
@@ -1403,5 +1404,18 @@ public final class ProtobufUtil {
}
}
+ /*
+ * Get the total (read + write) requests from a RegionLoad pb
+ * @param rl - RegionLoad pb
+ * @return total (read + write) requests
+ */
+ public static long getTotalRequestsCount(RegionLoad rl) {
+ if (rl == null) {
+ return 0;
+ }
+
+ return rl.getReadRequestsCount() + rl.getWriteRequestsCount();
+ }
+
// End helpers for Admin
}