svn commit: r1101472 - in /hive/trunk/ql/src/java/org/apache/hadoop/hive/ql: Driver.java lockmgr/zookeeper/ZooKeeperHiveLockManager.java

[nz...@apache.org]

Author: nzhang
Date: Tue May 10 14:30:56 2011
New Revision: 1101472

URL: http://svn.apache.org/viewvc?rev=1101472&view=rev
Log:
HIVE-2151. Too many open files in running negative cli tests (Yongqiang He via Ning Zhang)

Modified:
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ZooKeeperHiveLockManager.java

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java?rev=1101472&r1=1101471&r2=1101472&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java Tue May 10 14:30:56 2011
@@ -145,7 +145,11 @@ public class Driver implements CommandPr
         return false;
       }
     }
-    return true;
+    // the reason that we set the lock manager for the cxt here is because each
+    // query has its own ctx object. The hiveLockMgr is shared accross the
+    // same instance of Driver, which can run multiple queries.
+    ctx.setHiveLockMgr(hiveLockMgr);
+    return hiveLockMgr != null;
   }
 
   private void setLockManager() throws SemanticException {
@@ -161,11 +165,21 @@ public class Driver implements CommandPr
             conf);
         hiveLockMgr.setContext(new HiveLockManagerCtx(conf));
       } catch (Exception e) {
+        // set hiveLockMgr to null just in case this invalid manager got set to
+        // next query's ctx.  
+        if (hiveLockMgr != null) {
+          try {
+            hiveLockMgr.close();
+          } catch (LockException e1) {
+            //nothing can do here
+          }
+          hiveLockMgr = null;          
+        }
         throw new SemanticException(ErrorMsg.LOCKMGR_NOT_INITIALIZED.getMsg() + e.getMessage());
       }
     }
   }
-
+  
   public void init() {
     Operator.resetId();
   }
@@ -827,17 +841,15 @@ public class Driver implements CommandPr
     SQLState = null;
 
     int ret = compile(command);
-
-    boolean requireLock = false;
-    boolean ckLock = checkLockManager();
-
     if (ret != 0) {
       releaseLocks(ctx.getHiveLocks());
       return new CommandProcessorResponse(ret, errorMessage, SQLState);
     }
-    
+
+    boolean requireLock = false;
+    boolean ckLock = checkLockManager();
+
     if (ckLock) {
-      ctx.setHiveLockMgr(hiveLockMgr);
       boolean lockOnlyMapred = HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_LOCK_MAPRED_ONLY);
       if(lockOnlyMapred) {
         Queue<Task<? extends Serializable>> taskQueue = new LinkedList<Task<? extends Serializable>>();

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ZooKeeperHiveLockManager.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ZooKeeperHiveLockManager.java?rev=1101472&r1=1101471&r2=1101472&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ZooKeeperHiveLockManager.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ZooKeeperHiveLockManager.java Tue May 10 14:30:56 2011
@@ -370,10 +370,11 @@ public class ZooKeeperHiveLockManager im
 
   /* Release all locks - including PERSISTENT locks */
   public static void releaseAllLocks(HiveConf conf) throws Exception {
+    ZooKeeper zkpClient = null;
     try {
       int sessionTimeout = conf.getIntVar(HiveConf.ConfVars.HIVE_ZOOKEEPER_SESSION_TIMEOUT);
       String quorumServers = getQuorumServers(conf);
-      ZooKeeper zkpClient = new ZooKeeper(quorumServers, sessionTimeout, new DummyWatcher());
+      zkpClient = new ZooKeeper(quorumServers, sessionTimeout, new DummyWatcher());
       String parent = conf.getVar(HiveConf.ConfVars.HIVE_ZOOKEEPER_NAMESPACE);
       List<HiveLock> locks = getLocks(conf, zkpClient, null, parent, false, false);
 
@@ -382,12 +383,14 @@ public class ZooKeeperHiveLockManager im
           unlock(conf, zkpClient, lock, parent);
         }
       }
-
-      zkpClient.close();
-      zkpClient = null;
     } catch (Exception e) {
       LOG.error("Failed to release all locks: " + e.getMessage());
       throw new Exception(ErrorMsg.ZOOKEEPER_CLIENT_COULD_NOT_BE_INITIALIZED.getMsg());
+    } finally {
+      if (zkpClient != null) {
+        zkpClient.close();
+        zkpClient = null;
+      }
     }
   }
 
@@ -490,6 +493,10 @@ public class ZooKeeperHiveLockManager im
     try {
       renewZookeeperInstance(sessionTimeout, quorumServers);
       checkRedundantNode("/" + parent);
+      if (zooKeeper != null) {
+        zooKeeper.close();
+        zooKeeper = null;
+      }
     } catch (Exception e) {
       // ignore all errors
     }
@@ -519,14 +526,16 @@ public class ZooKeeperHiveLockManager im
   /* Release all transient locks, by simply closing the client */
   public void close() throws LockException {
     try {
+
       if (zooKeeper != null) {
         zooKeeper.close();
         zooKeeper = null;
       }
-
+      
       if (HiveConf.getBoolVar(ctx.getConf(), HiveConf.ConfVars.HIVE_ZOOKEEPER_CLEAN_EXTRA_NODES)) {
         removeAllRedundantNodes();
       }
+
     } catch (Exception e) {
       LOG.error("Failed to close zooKeeper client: " + e);
       throw new LockException(e);