You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by he...@apache.org on 2012/01/11 04:44:17 UTC
svn commit: r1229885 - in /hive/trunk/ql/src/java/org/apache/hadoop/hive/ql:
exec/Utilities.java processors/AddResourceProcessor.java
processors/DeleteResourceProcessor.java session/SessionState.java
Author: heyongqiang
Date: Wed Jan 11 03:44:17 2012
New Revision: 1229885
URL: http://svn.apache.org/viewvc?rev=1229885&view=rev
Log:
HIVE-2706 [jira] StackOverflowError when using custom UDF after adding archive
after adding jars
(Kevin Wilfong via Yongqiang He)
Summary:
https://issues.apache.org/jira/browse/HIVE-2706
The issue was that the current thread's classloader and the classloader in the
conf differed due to the prehook updating only the current thread's classloader
with new jars. Now, it updates both classloaders, fixing the issue.
When a custom UDF is used in a query after add an archive, such as a zip file,
after adding jars, the XMLEncoder enters an infinite loop when serializing the
map reduce task, as part of sending it to be executed. This results in a stack
overflow error.
Test Plan: Verified it fixed the stack overflow error.
Reviewers: JIRA, heyongqiang, njain
Reviewed By: heyongqiang
CC: heyongqiang
Differential Revision: https://reviews.facebook.net/D1167
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/processors/AddResourceProcessor.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/processors/DeleteResourceProcessor.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java?rev=1229885&r1=1229884&r2=1229885&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java Wed Jan 11 03:44:17 2012
@@ -120,6 +120,7 @@ import org.apache.hadoop.hive.ql.plan.Pa
import org.apache.hadoop.hive.ql.plan.PlanUtils;
import org.apache.hadoop.hive.ql.plan.PlanUtils.ExpressionTypes;
import org.apache.hadoop.hive.ql.plan.TableDesc;
+import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.ql.stats.StatsFactory;
import org.apache.hadoop.hive.ql.stats.StatsPublisher;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
@@ -144,7 +145,6 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.SequenceFileInputFormat;
import org.apache.hadoop.mapred.SequenceFileOutputFormat;
import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.hadoop.hive.ql.session.SessionState;
/**
* Utilities.
@@ -1501,18 +1501,6 @@ public final class Utilities {
return new URLClassLoader(curPath.toArray(new URL[0]), loader);
}
- public static ClassLoader addResourceFilesToClassPath(Configuration conf) {
- try {
- String addedJars = getResourceFiles(conf, SessionState.ResourceType.JAR);
- if (!StringUtils.isNotBlank(addedJars)) {
- return conf.getClassLoader();
- }
- return addToClassPath(conf.getClassLoader(), StringUtils.split(addedJars, ','));
- } catch (Exception e) {
- throw new RuntimeException("Error in adding jars to conf ", e);
- }
- }
-
/**
* remove elements from the classpath.
*
@@ -1536,6 +1524,7 @@ public final class Utilities {
loader = new URLClassLoader(newPath.toArray(new URL[0]));
curThread.setContextClassLoader(loader);
+ SessionState.get().getConf().setClassLoader(loader);
}
public static String formatBinaryString(byte[] array, int start, int length) {
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/processors/AddResourceProcessor.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/processors/AddResourceProcessor.java?rev=1229885&r1=1229884&r2=1229885&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/processors/AddResourceProcessor.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/processors/AddResourceProcessor.java Wed Jan 11 03:44:17 2012
@@ -21,7 +21,6 @@ package org.apache.hadoop.hive.ql.proces
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.parse.VariableSubstitution;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
@@ -59,9 +58,6 @@ public class AddResourceProcessor implem
}
}
- Thread.currentThread().setContextClassLoader(
- Utilities.addResourceFilesToClassPath(ss.getConf()));
-
return new CommandProcessorResponse(0);
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/processors/DeleteResourceProcessor.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/processors/DeleteResourceProcessor.java?rev=1229885&r1=1229884&r2=1229885&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/processors/DeleteResourceProcessor.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/processors/DeleteResourceProcessor.java Wed Jan 11 03:44:17 2012
@@ -21,7 +21,6 @@ package org.apache.hadoop.hive.ql.proces
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.parse.VariableSubstitution;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
@@ -60,9 +59,6 @@ public class DeleteResourceProcessor imp
ss.delete_resource(t);
}
- Thread.currentThread().setContextClassLoader(
- Utilities.addResourceFilesToClassPath(ss.getConf()));
-
return new CommandProcessorResponse(0);
}
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java?rev=1229885&r1=1229884&r2=1229885&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java Wed Jan 11 03:44:17 2012
@@ -418,8 +418,9 @@ public class SessionState {
LogHelper console = getConsole();
try {
ClassLoader loader = Thread.currentThread().getContextClassLoader();
- Thread.currentThread().setContextClassLoader(
- Utilities.addToClassPath(loader, StringUtils.split(newJar, ",")));
+ ClassLoader newLoader = Utilities.addToClassPath(loader, StringUtils.split(newJar, ","));
+ Thread.currentThread().setContextClassLoader(newLoader);
+ SessionState.get().getConf().setClassLoader(newLoader);
console.printInfo("Added " + newJar + " to class path");
return true;
} catch (Exception e) {