You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by he...@apache.org on 2011/12/09 19:37:25 UTC
svn commit: r1212567 - in /hive/trunk/ql/src/java/org/apache/hadoop/hive/ql:
exec/ExecDriver.java exec/MapRedTask.java exec/MapredLocalTask.java
exec/Utilities.java io/rcfile/merge/BlockMergeTask.java
Author: heyongqiang
Date: Fri Dec 9 18:37:25 2011
New Revision: 1212567
URL: http://svn.apache.org/viewvc?rev=1212567&view=rev
Log:
HIVE-2635 [jira] wrong class loader used for external jars
(Namit Jain via Yongqiang He)
Summary:
HIVE-2635
Test Plan: EMPTY
Reviewers: JIRA, heyongqiang
Reviewed By: heyongqiang
CC: heyongqiang
Differential Revision: 687
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapRedTask.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapredLocalTask.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/BlockMergeTask.java
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java?rev=1212567&r1=1212566&r2=1212567&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java Fri Dec 9 18:37:25 2011
@@ -116,26 +116,6 @@ public class ExecDriver extends Task<Map
return true;
}
- public static String getResourceFiles(Configuration conf, SessionState.ResourceType t) {
- // fill in local files to be added to the task environment
- SessionState ss = SessionState.get();
- Set<String> files = (ss == null) ? null : ss.list_resource(t, null);
- if (files != null) {
- List<String> realFiles = new ArrayList<String>(files.size());
- for (String one : files) {
- try {
- realFiles.add(Utilities.realFile(one, conf));
- } catch (IOException e) {
- throw new RuntimeException("Cannot validate file " + one + "due to exception: "
- + e.getMessage(), e);
- }
- }
- return StringUtils.join(realFiles, ",");
- } else {
- return "";
- }
- }
-
private void initializeFiles(String prop, String files) {
if (files != null && files.length() > 0) {
job.set(prop, files);
@@ -159,15 +139,15 @@ public class ExecDriver extends Task<Map
//
// "tmpfiles" and "tmpjars" are set by the method ExecDriver.execute(),
// which will be called by both local and NON-local mode.
- String addedFiles = getResourceFiles(job, SessionState.ResourceType.FILE);
+ String addedFiles = Utilities.getResourceFiles(job, SessionState.ResourceType.FILE);
if (StringUtils.isNotBlank(addedFiles)) {
HiveConf.setVar(job, ConfVars.HIVEADDEDFILES, addedFiles);
}
- String addedJars = getResourceFiles(job, SessionState.ResourceType.JAR);
+ String addedJars = Utilities.getResourceFiles(job, SessionState.ResourceType.JAR);
if (StringUtils.isNotBlank(addedJars)) {
HiveConf.setVar(job, ConfVars.HIVEADDEDJARS, addedJars);
}
- String addedArchives = getResourceFiles(job, SessionState.ResourceType.ARCHIVE);
+ String addedArchives = Utilities.getResourceFiles(job, SessionState.ResourceType.ARCHIVE);
if (StringUtils.isNotBlank(addedArchives)) {
HiveConf.setVar(job, ConfVars.HIVEADDEDARCHIVES, addedArchives);
}
@@ -777,7 +757,7 @@ public class ExecDriver extends Task<Map
public Collection<Operator<? extends Serializable>> getTopOperators() {
return getWork().getAliasToWork().values();
}
-
+
@Override
public boolean hasReduce() {
MapredWork w = getWork();
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapRedTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapRedTask.java?rev=1212567&r1=1212566&r2=1212567&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapRedTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapRedTask.java Fri Dec 9 18:37:25 2011
@@ -147,7 +147,7 @@ public class MapRedTask extends ExecDriv
String hiveJar = conf.getJar();
String libJarsOption;
- String addedJars = getResourceFiles(conf, SessionState.ResourceType.JAR);
+ String addedJars = Utilities.getResourceFiles(conf, SessionState.ResourceType.JAR);
conf.setVar(ConfVars.HIVEADDEDJARS, addedJars);
String auxJars = conf.getAuxJars();
// Put auxjars and addedjars together into libjars
@@ -188,7 +188,7 @@ public class MapRedTask extends ExecDriv
+ planPath.toString() + " " + isSilent + " " + hiveConfArgs;
String workDir = (new File(".")).getCanonicalPath();
- String files = getResourceFiles(conf, SessionState.ResourceType.FILE);
+ String files = Utilities.getResourceFiles(conf, SessionState.ResourceType.FILE);
if (!files.isEmpty()) {
cmdLine = cmdLine + " -files " + files;
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapredLocalTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapredLocalTask.java?rev=1212567&r1=1212566&r2=1212567&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapredLocalTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapredLocalTask.java Fri Dec 9 18:37:25 2011
@@ -134,7 +134,7 @@ public class MapredLocalTask extends Tas
+ " " + isSilent + " " + hiveConfArgs;
String workDir = (new File(".")).getCanonicalPath();
- String files = ExecDriver.getResourceFiles(conf, SessionState.ResourceType.FILE);
+ String files = Utilities.getResourceFiles(conf, SessionState.ResourceType.FILE);
if (!files.isEmpty()) {
cmdLine = cmdLine + " -files " + files;
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java?rev=1212567&r1=1212566&r2=1212567&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java Fri Dec 9 18:37:25 2011
@@ -144,6 +144,7 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.SequenceFileInputFormat;
import org.apache.hadoop.mapred.SequenceFileOutputFormat;
import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.hive.ql.session.SessionState;
/**
* Utilities.
@@ -403,8 +404,11 @@ public final class Utilities {
} catch (UnsupportedEncodingException ex) {
throw new RuntimeException("UTF-8 support required", ex);
}
+
ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
- XMLDecoder decoder = new XMLDecoder(bais, null, null, conf.getClassLoader());
+
+ XMLDecoder decoder = new XMLDecoder(bais, null, null,
+ addResourceFilesToClassPath(conf));
try {
ExprNodeDesc expr = (ExprNodeDesc) decoder.readObject();
return expr;
@@ -477,7 +481,8 @@ public final class Utilities {
public static QueryPlan deserializeQueryPlan(InputStream in, Configuration conf) {
XMLDecoder d = null;
try {
- d = new XMLDecoder(in, null, null, conf.getClassLoader());
+ d = new XMLDecoder(in, null, null,
+ addResourceFilesToClassPath(conf));
QueryPlan ret = (QueryPlan) d.readObject();
return (ret);
} finally {
@@ -510,7 +515,8 @@ public final class Utilities {
public static MapredWork deserializeMapRedWork(InputStream in, Configuration conf) {
XMLDecoder d = null;
try {
- d = new XMLDecoder(in, null, null, conf.getClassLoader());
+ d = new XMLDecoder(in, null, null,
+ addResourceFilesToClassPath(conf));
MapredWork ret = (MapredWork) d.readObject();
return (ret);
} finally {
@@ -542,7 +548,8 @@ public final class Utilities {
public static MapredLocalWork deserializeMapRedLocalWork(InputStream in, Configuration conf) {
XMLDecoder d = null;
try {
- d = new XMLDecoder(in, null, null, conf.getClassLoader());
+ d = new XMLDecoder(in, null, null,
+ addResourceFilesToClassPath(conf));
MapredLocalWork ret = (MapredLocalWork) d.readObject();
return (ret);
} finally {
@@ -1446,6 +1453,26 @@ public final class Utilities {
return e.getClass().getName() + "(" + e.getMessage() + ")";
}
+ public static String getResourceFiles(Configuration conf, SessionState.ResourceType t) {
+ // fill in local files to be added to the task environment
+ SessionState ss = SessionState.get();
+ Set<String> files = (ss == null) ? null : ss.list_resource(t, null);
+ if (files != null) {
+ List<String> realFiles = new ArrayList<String>(files.size());
+ for (String one : files) {
+ try {
+ realFiles.add(realFile(one, conf));
+ } catch (IOException e) {
+ throw new RuntimeException("Cannot validate file " + one + "due to exception: "
+ + e.getMessage(), e);
+ }
+ }
+ return StringUtils.join(realFiles, ",");
+ } else {
+ return "";
+ }
+ }
+
/**
* Add new elements to the classpath.
*
@@ -1478,6 +1505,18 @@ public final class Utilities {
return new URLClassLoader(curPath.toArray(new URL[0]), loader);
}
+ public static ClassLoader addResourceFilesToClassPath(Configuration conf) {
+ try {
+ String addedJars = getResourceFiles(conf, SessionState.ResourceType.JAR);
+ if (!StringUtils.isNotBlank(addedJars)) {
+ return conf.getClassLoader();
+ }
+ return addToClassPath(conf.getClassLoader(), StringUtils.split(addedJars, ','));
+ } catch (Exception e) {
+ throw new RuntimeException("Error in adding jars to conf ", e);
+ }
+ }
+
/**
* remove elements from the classpath.
*
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/BlockMergeTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/BlockMergeTask.java?rev=1212567&r1=1212566&r2=1212567&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/BlockMergeTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/BlockMergeTask.java Fri Dec 9 18:37:25 2011
@@ -192,8 +192,8 @@ public class BlockMergeTask extends Task
HiveConf.setVar(job, HiveConf.ConfVars.METASTOREPWD, "HIVE");
}
JobClient jc = new JobClient(job);
-
- String addedJars = ExecDriver.getResourceFiles(job, SessionState.ResourceType.JAR);
+
+ String addedJars = Utilities.getResourceFiles(job, SessionState.ResourceType.JAR);
if (!addedJars.isEmpty()) {
job.set("tmpjars", addedJars);
}