You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by pr...@apache.org on 2015/05/12 03:24:07 UTC
[21/39] hive git commit: HIVE-10453 : HS2 leaking open file
descriptors when using UDFs (Yongzhi Chen via Szehon)
HIVE-10453 : HS2 leaking open file descriptors when using UDFs (Yongzhi Chen via Szehon)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/42f88ca9
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/42f88ca9
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/42f88ca9
Branch: refs/heads/llap
Commit: 42f88ca9a105956a2085b75dae681d62dd784cef
Parents: 6176333
Author: Szehon Ho <sz...@cloudera.com>
Authored: Thu May 7 11:20:53 2015 -0700
Committer: Szehon Ho <sz...@cloudera.com>
Committed: Thu May 7 11:20:53 2015 -0700
----------------------------------------------------------------------
.../apache/hadoop/hive/ql/exec/Registry.java | 29 ++++++++++++++++++--
.../apache/hadoop/hive/ql/exec/Utilities.java | 5 ++++
.../hadoop/hive/ql/session/SessionState.java | 4 +--
3 files changed, 34 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/42f88ca9/ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java
index 2ba91d0..a5d59ae 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java
@@ -20,8 +20,10 @@ package org.apache.hadoop.hive.ql.exec;
import com.google.common.base.Splitter;
import com.google.common.collect.Sets;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.common.JavaUtils;
import org.apache.hadoop.hive.ql.exec.FunctionInfo.FunctionResource;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -43,9 +45,12 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.util.ReflectionUtils;
+import java.io.IOException;
+import java.net.URLClassLoader;
import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -66,6 +71,7 @@ public class Registry {
*/
private final Map<String, FunctionInfo> mFunctions = new LinkedHashMap<String, FunctionInfo>();
private final Set<Class<?>> builtIns = Collections.synchronizedSet(new HashSet<Class<?>>());
+ private final Set<ClassLoader> mSessionUDFLoaders = new LinkedHashSet<ClassLoader>();
private final boolean isNative;
@@ -443,7 +449,6 @@ public class Registry {
// should be called after session registry is checked
private FunctionInfo registerToSessionRegistry(String qualifiedName, FunctionInfo function) {
FunctionInfo ret = null;
-
ClassLoader prev = Utilities.getSessionSpecifiedClassLoader();
try {
// Found UDF in metastore - now add it to the function registry
@@ -455,7 +460,6 @@ public class Registry {
LOG.error("Unable to load resources for " + qualifiedName + ":" + e, e);
return null;
}
-
ClassLoader loader = Utilities.getSessionSpecifiedClassLoader();
Class<?> udfClass = Class.forName(function.getClassName(), true, loader);
@@ -463,6 +467,9 @@ public class Registry {
if (ret == null) {
LOG.error(function.getClassName() + " is not a valid UDF class and was not registered.");
}
+ if (SessionState.get().isHiveServerQuery()) {
+ SessionState.getRegistryForWrite().addToUDFLoaders(loader);
+ }
} catch (ClassNotFoundException e) {
// Lookup of UDf class failed
LOG.error("Unable to load UDF class: " + e);
@@ -489,6 +496,24 @@ public class Registry {
builtIns.clear();
}
+ public synchronized void closeCUDFLoaders() {
+ try {
+ for(ClassLoader loader: mSessionUDFLoaders) {
+ JavaUtils.closeClassLoader(loader);
+ }
+ } catch (IOException ie) {
+ LOG.error("Error in close loader: " + ie);
+ }
+ mSessionUDFLoaders.clear();
+ }
+
+ public synchronized void addToUDFLoaders(ClassLoader loader) {
+ mSessionUDFLoaders.add(loader);
+ }
+ public synchronized void removeFromUDFLoaders(ClassLoader loader) {
+ mSessionUDFLoaders.remove(loader);
+ }
+
/**
* Setup blocked flag for all builtin UDFs as per udf whitelist and blacklist
* @param whiteListStr
http://git-wip-us.apache.org/repos/asf/hive/blob/42f88ca9/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
index ad5c8f8..7b48b8b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
@@ -2250,6 +2250,11 @@ public final class Utilities {
}
}
JavaUtils.closeClassLoader(loader);
+//this loader is closed, remove it from cached registry loaders to avoid remove it again.
+ Registry reg = SessionState.getRegistry();
+ if(reg != null) {
+ reg.removeFromUDFLoaders(loader);
+ }
loader = new URLClassLoader(newPath.toArray(new URL[0]));
curThread.setContextClassLoader(loader);
http://git-wip-us.apache.org/repos/asf/hive/blob/42f88ca9/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
index 8db78e5..37b6d6f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
@@ -1458,7 +1458,7 @@ public class SessionState {
}
public void close() throws IOException {
- registry.clear();;
+ registry.clear();
if (txnMgr != null) txnMgr.closeTxnManager();
JavaUtils.closeClassLoadersTo(conf.getClassLoader(), parentLoader);
File resourceDir =
@@ -1493,7 +1493,7 @@ public class SessionState {
sparkSession = null;
}
}
-
+ registry.closeCUDFLoaders();
dropSessionPaths(conf);
}