svn commit: r1504218 - in /hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase: client/HConnectionManager.java mapreduce/TableInputFormatBase.java

[li...@apache.org]

Author: liyin
Date: Wed Jul 17 18:21:58 2013
New Revision: 1504218

URL: http://svn.apache.org/r1504218
Log:
[HBASE-8905] Closes the connections after the split is computed

Author: rshroff

Summary:
Currently in the TableInputFormatBase, after getting the input splits,
it is not closing the connections to the client. Most of the time this
is fine as we don't trigger to many jobs from a particular node.

However, there is per client limit on the number of connections in the
Zookeeper and in case we trigger more jobs from a node, it will run to
errors.

Test Plan:
test on SH004. Started job and checked the logs to make sure,
it closes the connection.

P2303750

Reviewers: liyintang

Reviewed By: liyintang

CC: hbase-eng@, chaoyc

Differential Revision: https://phabricator.fb.com/D873353

Task ID: 2558905

Modified:
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java?rev=1504218&r1=1504217&r2=1504218&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java Wed Jul 17 18:21:58 2013
@@ -177,7 +177,6 @@ public class HConnectionManager {
 
   /**
    * Delete information for all connections.
-   * @param stopProxy stop the proxy as well
    */
   public static void deleteAllConnections() {
     synchronized (HBASE_INSTANCES) {
@@ -188,10 +187,18 @@ public class HConnectionManager {
       }
       HBaseRPC.stopClients ();
     }
+    deleteAllZookeeperConnections();
+  }
+
+  /**
+   * Delete information for all zookeeper connections.
+   */
+  public static void deleteAllZookeeperConnections() {
     synchronized (ZK_WRAPPERS) {
       for (ClientZKConnection connection : ZK_WRAPPERS.values()) {
         connection.closeZooKeeperConnection();
       }
+      ZK_WRAPPERS.clear();
     }
   }
 

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java?rev=1504218&r1=1504217&r2=1504218&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java Wed Jul 17 18:21:58 2013
@@ -34,6 +34,7 @@ import org.apache.commons.logging.LogFac
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HServerAddress;
+import org.apache.hadoop.hbase.client.HConnectionManager;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.Scan;
@@ -220,6 +221,8 @@ extends InputFormat<ImmutableBytesWritab
           LOG.debug("getSplits: split -> " + i + " -> " + split);
       }
     }
+
+    HConnectionManager.deleteAllZookeeperConnections();
     return splits;
   }