You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2014/03/23 17:54:47 UTC
svn commit: r1580532 - in /hive/branches/branch-0.13:
common/src/java/org/apache/hadoop/hive/common/
ql/src/java/org/apache/hadoop/hive/ql/exec/
ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/
ql/src/java/org/apache/hadoop/hive/ql/session/ ql/...
Author: hashutosh
Date: Sun Mar 23 16:54:47 2014
New Revision: 1580532
URL: http://svn.apache.org/r1580532
Log:
HIVE-3969 : Session state for hive server should be cleaned-up (Navis via Ashutosh Chauhan)
Modified:
hive/branches/branch-0.13/common/src/java/org/apache/hadoop/hive/common/JavaUtils.java
hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java
hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/LineageCtx.java
hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionEvaluator.java
hive/branches/branch-0.13/ql/src/test/org/apache/hadoop/hive/ql/session/TestSessionState.java
hive/branches/branch-0.13/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFOPNumeric.java
hive/branches/branch-0.13/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java
hive/branches/branch-0.13/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
Modified: hive/branches/branch-0.13/common/src/java/org/apache/hadoop/hive/common/JavaUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/common/src/java/org/apache/hadoop/hive/common/JavaUtils.java?rev=1580532&r1=1580531&r2=1580532&view=diff
==============================================================================
--- hive/branches/branch-0.13/common/src/java/org/apache/hadoop/hive/common/JavaUtils.java (original)
+++ hive/branches/branch-0.13/common/src/java/org/apache/hadoop/hive/common/JavaUtils.java Sun Mar 23 16:54:47 2014
@@ -18,12 +18,40 @@
package org.apache.hadoop.hive.common;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.net.URLClassLoader;
+import java.util.Arrays;
+
/**
* Collection of Java class loading/reflection related utilities common across
* Hive.
*/
public final class JavaUtils {
+ private static final Log LOG = LogFactory.getLog(JavaUtils.class);
+ private static final Method SUN_MISC_UTIL_RELEASE;
+
+ static {
+ if (Closeable.class.isAssignableFrom(URLClassLoader.class)) {
+ SUN_MISC_UTIL_RELEASE = null;
+ } else {
+ Method release = null;
+ try {
+ Class<?> clazz = Class.forName("sun.misc.ClassLoaderUtil");
+ release = clazz.getMethod("releaseLoader", URLClassLoader.class);
+ } catch (Exception e) {
+ // ignore
+ }
+ SUN_MISC_UTIL_RELEASE = release;
+ }
+ }
+
/**
* Standard way of getting classloader in Hive code (outside of Hadoop).
*
@@ -40,6 +68,49 @@ public final class JavaUtils {
return classLoader;
}
+ public static void closeClassLoadersTo(ClassLoader current, ClassLoader stop) {
+ if (!isValidHierarchy(current, stop)) {
+ return;
+ }
+ for (; current != null && current != stop; current = current.getParent()) {
+ try {
+ closeClassLoader(current);
+ } catch (IOException e) {
+ LOG.info("Failed to close class loader " + current +
+ Arrays.toString(((URLClassLoader) current).getURLs()), e);
+ }
+ }
+ }
+
+ // check before closing loaders, not to close app-classloader, etc. by mistake
+ private static boolean isValidHierarchy(ClassLoader current, ClassLoader stop) {
+ if (current == null || stop == null || current == stop) {
+ return false;
+ }
+ for (; current != null && current != stop; current = current.getParent()) {
+ }
+ return current == stop;
+ }
+
+ // best effort to close
+ // see https://issues.apache.org/jira/browse/HIVE-3969 for detail
+ public static void closeClassLoader(ClassLoader loader) throws IOException {
+ if (loader instanceof Closeable) {
+ ((Closeable)loader).close();
+ } else if (SUN_MISC_UTIL_RELEASE != null && loader instanceof URLClassLoader) {
+ try {
+ SUN_MISC_UTIL_RELEASE.invoke(null, loader);
+ } catch (InvocationTargetException e) {
+ if (e.getTargetException() instanceof IOException) {
+ throw (IOException)e.getTargetException();
+ }
+ throw new IOException(e.getTargetException());
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
+ }
+
private JavaUtils() {
// prevent instantiation
}
Modified: hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java?rev=1580532&r1=1580531&r2=1580532&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java (original)
+++ hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java Sun Mar 23 16:54:47 2014
@@ -96,7 +96,14 @@ public class PTFOperator extends Operato
}
}
inputPart.close();
- }
+ inputPart = null;
+
+ for (PTFInputDef iDef = conf.getFuncDef(); iDef != null; iDef = iDef.getInput()) {
+ if (iDef instanceof PartitionedTableFunctionDef) {
+ ((PartitionedTableFunctionDef)iDef).getTFunction().close();
+ }
+ }
+ }
@Override
public void processOp(Object row, int tag) throws HiveException
Modified: hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/LineageCtx.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/LineageCtx.java?rev=1580532&r1=1580531&r2=1580532&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/LineageCtx.java (original)
+++ hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/LineageCtx.java Sun Mar 23 16:54:47 2014
@@ -123,6 +123,10 @@ public class LineageCtx implements NodeP
old_dep.setExpr(null);
}
}
+
+ public void clear() {
+ depMap.clear();
+ }
}
/**
Modified: hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java?rev=1580532&r1=1580531&r2=1580532&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java (original)
+++ hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java Sun Mar 23 16:54:47 2014
@@ -40,6 +40,7 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.common.JavaUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.ql.MapRedStats;
@@ -74,6 +75,8 @@ import org.apache.hadoop.util.Reflection
public class SessionState {
private static final Log LOG = LogFactory.getLog(SessionState.class);
+ protected ClassLoader parentLoader;
+
/**
* current configuration.
*/
@@ -237,6 +240,7 @@ public class SessionState {
if (StringUtils.isEmpty(conf.getVar(HiveConf.ConfVars.HIVESESSIONID))) {
conf.setVar(HiveConf.ConfVars.HIVESESSIONID, makeSessionId());
}
+ parentLoader = JavaUtils.getClassLoader();
}
private static final SimpleDateFormat DATE_FORMAT =
@@ -286,8 +290,13 @@ public class SessionState {
/**
* Sets the given session state in the thread local var for sessions.
*/
- public static void setCurrentSessionState(SessionState session) {
- tss.set(session);
+ public static void setCurrentSessionState(SessionState startSs) {
+ tss.set(startSs);
+ Thread.currentThread().setContextClassLoader(startSs.getConf().getClassLoader());
+ }
+
+ public static void detachSession() {
+ tss.remove();
}
/**
@@ -930,6 +939,7 @@ public class SessionState {
}
public void close() throws IOException {
+ JavaUtils.closeClassLoadersTo(conf.getClassLoader(), parentLoader);
File resourceDir =
new File(getConf().getVar(HiveConf.ConfVars.DOWNLOADED_RESOURCES_DIR));
LOG.debug("Removing resource dir " + resourceDir);
@@ -939,6 +949,8 @@ public class SessionState {
}
} catch (IOException e) {
LOG.info("Error removing session resource dir " + resourceDir, e);
+ } finally {
+ detachSession();
}
try {
Modified: hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionEvaluator.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionEvaluator.java?rev=1580532&r1=1580531&r2=1580532&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionEvaluator.java (original)
+++ hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionEvaluator.java Sun Mar 23 16:54:47 2014
@@ -137,5 +137,12 @@ public abstract class TableFunctionEvalu
protected PTFPartition _transformRawInput(PTFPartition iPart) throws HiveException {
return null;
}
+
+ public void close() {
+ if (outputPartition != null) {
+ outputPartition.close();
+ }
+ outputPartition = null;
+ }
}
Modified: hive/branches/branch-0.13/ql/src/test/org/apache/hadoop/hive/ql/session/TestSessionState.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/test/org/apache/hadoop/hive/ql/session/TestSessionState.java?rev=1580532&r1=1580531&r2=1580532&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/test/org/apache/hadoop/hive/ql/session/TestSessionState.java (original)
+++ hive/branches/branch-0.13/ql/src/test/org/apache/hadoop/hive/ql/session/TestSessionState.java Sun Mar 23 16:54:47 2014
@@ -88,4 +88,45 @@ public class TestSessionState {
ss.close();
assertNull(ss.getTezSession());
}
+
+ class RegisterJarRunnable implements Runnable {
+ String jar;
+ ClassLoader loader;
+ SessionState ss;
+
+ public RegisterJarRunnable(String jar, SessionState ss) {
+ this.jar = jar;
+ this.ss = ss;
+ }
+
+ public void run() {
+ SessionState.start(ss);
+ SessionState.registerJar(jar);
+ loader = Thread.currentThread().getContextClassLoader();
+ }
+ }
+
+ @Test
+ public void testClassLoaderEquality() throws Exception {
+ HiveConf conf = new HiveConf();
+ final SessionState ss1 = new SessionState(conf);
+ RegisterJarRunnable otherThread = new RegisterJarRunnable("./build/contrib/test/test-udfs.jar", ss1);
+ Thread th1 = new Thread(otherThread);
+ th1.start();
+ th1.join();
+
+ // set state in current thread
+ SessionState.start(ss1);
+ SessionState ss2 = SessionState.get();
+ ClassLoader loader2 = ss2.conf.getClassLoader();
+
+ System.out.println("Loader1:(Set in other thread) " + otherThread.loader);
+ System.out.println("Loader2:(Set in SessionState.conf) " + loader2);
+ System.out.println("Loader3:(CurrentThread.getContextClassLoader()) " +
+ Thread.currentThread().getContextClassLoader());
+ assertEquals("Other thread loader and session state loader",
+ otherThread.loader, loader2);
+ assertEquals("Other thread loader and current thread loader",
+ otherThread.loader, Thread.currentThread().getContextClassLoader());
+ }
}
Modified: hive/branches/branch-0.13/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFOPNumeric.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFOPNumeric.java?rev=1580532&r1=1580531&r2=1580532&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFOPNumeric.java (original)
+++ hive/branches/branch-0.13/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFOPNumeric.java Sun Mar 23 16:54:47 2014
@@ -8,7 +8,6 @@ import org.apache.hadoop.hive.serde2.obj
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
-import org.apache.hive.common.HiveCompat;
import org.junit.Assert;
public abstract class TestGenericUDFOPNumeric {
Modified: hive/branches/branch-0.13/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java?rev=1580532&r1=1580531&r2=1580532&view=diff
==============================================================================
--- hive/branches/branch-0.13/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java (original)
+++ hive/branches/branch-0.13/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java Sun Mar 23 16:54:47 2014
@@ -204,6 +204,7 @@ public class SQLOperation extends Execut
driver.close();
driver.destroy();
}
+ driver = null;
SessionState ss = SessionState.get();
if (ss.getTmpOutputFile() != null) {
Modified: hive/branches/branch-0.13/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java?rev=1580532&r1=1580531&r2=1580532&view=diff
==============================================================================
--- hive/branches/branch-0.13/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java (original)
+++ hive/branches/branch-0.13/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java Sun Mar 23 16:54:47 2014
@@ -35,7 +35,6 @@ import org.apache.hadoop.hive.ql.exec.Fe
import org.apache.hadoop.hive.ql.exec.ListSinkOperator;
import org.apache.hadoop.hive.ql.history.HiveHistory;
import org.apache.hadoop.hive.ql.session.SessionState;
-import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hive.common.util.HiveVersionInfo;
import org.apache.hive.service.auth.HiveAuthFactory;
import org.apache.hive.service.cli.FetchOrientation;
@@ -144,7 +143,7 @@ public class HiveSessionImpl implements
protected synchronized void release() {
assert sessionState != null;
- // no need to release sessionState...
+ SessionState.detachSession();
}
@Override
@@ -411,10 +410,10 @@ public class HiveSessionImpl implements
hiveHist.closeStream();
}
sessionState.close();
- release();
} catch (IOException ioe) {
- release();
throw new HiveSQLException("Failure to close", ioe);
+ } finally {
+ release();
}
}