svn commit: r1202979 - /hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/RegionManager.java

[ns...@apache.org]

Author: nspiegelberg
Date: Thu Nov 17 01:44:12 2011
New Revision: 1202979

URL: http://svn.apache.org/viewvc?rev=1202979&view=rev
Log:
D352964 Remove unnecessary DNS lookups from HBase master

Summary: D352964 Remove unnecessary DNS lookups from HBase master

In titan015, we hit the issue where upon each RS heartbeat we were looking up and resolving DNS name + address of the RS in the master, but needed it only for locality based assignment on startup. Part of oncall, so keeping ready to hotfix if needed (diff against MARS branch).

Some flakiness in the DNS subsystem cause one of the threads to get stuck in the lookup and the synchronized call at:

ServerManager.java:528
processMsgs() {
...
synchronized (this.master.getRegionManager()) {
// does dns lookup
}
}

The offending stack trace was:

"IPC Server handler 232 on 60000" daemon prio=10 tid=0x00007fcb64164000 nid=0x7d16 runnable [0x0000000052e7f000]
java.lang.Thread.State: RUNNABLE
at java.net.Inet4AddressImpl.lookupAllHostAddr(Native Method)
at java.net.InetAddress$1.lookupAllHostAddr(InetAddress.java:849)
at java.net.InetAddress.getAddressFromNameService(InetAddress.java:1200)
at java.net.InetAddress.getAllByName0(InetAddress.java:1153)
at java.net.InetAddress.getAllByName0(InetAddress.java:1128)
at java.net.InetAddress.getHostFromNameService(InetAddress.java:550)
at java.net.InetAddress.getHostName(InetAddress.java:476)
at java.net.InetAddress.getHostName(InetAddress.java:448)
at java.net.InetSocketAddress.getHostName(InetSocketAddress.java:210)
at org.apache.hadoop.hbase.HServerAddress.getHostname(HServerAddress.java:117)
at org.apache.hadoop.hbase.master.RegionManager.regionsAwaitingAssignment(RegionManager.java:469)
at org.apache.hadoop.hbase.master.RegionManager.assignRegions(RegionManager.java:263)
at org.apache.hadoop.hbase.master.ServerManager.processMsgs(ServerManager.java:500)
- locked <0x00007fcb985b2030> (a org.apache.hadoop.hbase.master.RegionManager)
at org.apache.hadoop.hbase.master.ServerManager.processRegionServerAllsWell(ServerManager.java:425)
at org.apache.hadoop.hbase.master.ServerManager.regionServerReport(ServerManager.java:335)
at org.apache.hadoop.hbase.master.HMaster.regionServerReport(HMaster.java:841)
at sun.reflect.GeneratedMethodAccessor15.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:585)
at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:933)

Reviewed By:kannan

Test Plan:Tested on Titan clusters 15 and 16

Revert Plan:OK

Differential Revision: 352964

- begin *PUBLIC* platform impact section -
Bugzilla: #
- end platform impact -

Modified:
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/RegionManager.java

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/RegionManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/RegionManager.java?rev=1202979&r1=1202978&r2=1202979&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/RegionManager.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/RegionManager.java Thu Nov 17 01:44:12 2011
@@ -282,8 +282,6 @@ public class RegionManager {
 
     HServerLoad thisServersLoad = info.getLoad();
     boolean isSingleServer = this.master.numServers() == 1;
-    // have to add . at the end of host name
-    String hostName = info.getHostname();
     boolean holdRegionForBestRegionServer = false;
     boolean assignmentByLocality = false;
 
@@ -307,6 +305,8 @@ public class RegionManager {
     }
 
     if (assignmentByLocality) {
+      // have to add . at the end of host name
+      String hostName = info.getHostname();
       quickStartRegionServerSet.add(hostName);
     }
 
@@ -551,7 +551,8 @@ public class RegionManager {
     boolean isMetaServer = isMetaServer(addr);
     boolean isRootServer = isRootServer(addr);
     boolean isMetaOrRoot = isMetaServer || isRootServer;
-    String hostName = addr.getHostname();
+    // lookup hostname of addr if needed
+    String hostName = null;
     RegionState rootState = null;
     // Handle if root is unassigned... only assign root if root is offline.
     synchronized (this.regionsInTransition) {
@@ -609,6 +610,9 @@ public class RegionManager {
           Text preferredHostNameTxt =
             (Text)this.master.getPreferredRegionToRegionServerMapping().get(new Text(name));
 
+          if (hostName == null) {
+            hostName = addr.getHostname();
+          }
           if (preferredHostNameTxt != null) {
             String preferredHost = preferredHostNameTxt.toString();
             if (hostName.startsWith(preferredHost)) {