You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ek...@apache.org on 2014/11/06 21:20:31 UTC
svn commit: r1637207 - in
/hive/branches/branch-0.14/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton:
SqoopDelegator.java tool/LaunchMapper.java
Author: ekoifman
Date: Thu Nov 6 20:20:31 2014
New Revision: 1637207
URL: http://svn.apache.org/r1637207
Log:
HIVE-8754 Sqoop job submission via WebHCat doesn't properly localize required jdbc jars in secure cluster (Eugene Koifman, reviewed by Thejas Nair)
Modified:
hive/branches/branch-0.14/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SqoopDelegator.java
hive/branches/branch-0.14/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java
Modified: hive/branches/branch-0.14/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SqoopDelegator.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SqoopDelegator.java?rev=1637207&r1=1637206&r2=1637207&view=diff
==============================================================================
--- hive/branches/branch-0.14/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SqoopDelegator.java (original)
+++ hive/branches/branch-0.14/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SqoopDelegator.java Thu Nov 6 20:20:31 2014
@@ -83,6 +83,20 @@ public class SqoopDelegator extends Laun
args.add("-D" + TempletonControllerJob.TOKEN_FILE_ARG_PLACEHOLDER);
args.add("-D" + TempletonControllerJob.MAPREDUCE_JOB_TAGS_ARG_PLACEHOLDER);
}
+ if(i == 0 && TempletonUtils.isset(libdir) && TempletonUtils.isset(appConf.sqoopArchive())) {
+ //http://sqoop.apache.org/docs/1.4.5/SqoopUserGuide.html#_using_generic_and_specific_arguments
+ String libJars = null;
+ for(String s : args) {
+ if(s.startsWith(JobSubmissionConstants.Sqoop.LIB_JARS)) {
+ libJars = s.substring(s.indexOf("=") + 1);
+ break;
+ }
+ }
+ //the jars in libJars will be localized to CWD of the launcher task; then -libjars will
+ //cause them to be localized for the Sqoop MR job tasks
+ args.add(TempletonUtils.quoteForWindows("-libjars"));
+ args.add(TempletonUtils.quoteForWindows(libJars));
+ }
}
} else if (TempletonUtils.isset(optionsFile)) {
args.add("--options-file");
@@ -114,11 +128,13 @@ public class SqoopDelegator extends Laun
/**Sqoop accesses databases via JDBC. This means it needs to have appropriate JDBC
drivers available. Normally, the user would install Sqoop and place these jars
into SQOOP_HOME/lib. When WebHCat is configured to auto-ship the Sqoop tar file, we
- need to make sure that relevant JDBC jars are available on target node.
+ need to make sure that relevant JDBC jars are available on target node but we cannot modify
+ lib/ of exploded tar because Dist Cache intentionally prevents this.
The user is expected to place any JDBC jars into an HDFS directory and specify this
- dir in "libdir" parameter. All the files in this dir will be copied to lib/ of the
- exploded Sqoop tar ball on target node.
+ dir in "libdir" parameter. WebHCat then ensures that these jars are localized for the launcher task
+ and made available to Sqoop.
{@link org.apache.hive.hcatalog.templeton.tool.LaunchMapper#handleSqoop(org.apache.hadoop.conf.Configuration, java.util.Map)}
+ {@link #makeArgs(String, String, String, String, String, boolean, String)}
*/
LOG.debug("libdir=" + libdir);
List<Path> jarList = TempletonUtils.hadoopFsListChildren(libdir, appConf, runAs);
Modified: hive/branches/branch-0.14/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java?rev=1637207&r1=1637206&r2=1637207&view=diff
==============================================================================
--- hive/branches/branch-0.14/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java (original)
+++ hive/branches/branch-0.14/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java Thu Nov 6 20:20:31 2014
@@ -18,7 +18,6 @@
*/
package org.apache.hive.hcatalog.templeton.tool;
-import com.google.common.io.Files;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -101,11 +100,18 @@ public class LaunchMapper extends Mapper
if(TempletonUtils.isset(conf.get(Sqoop.LIB_JARS))) {
//LIB_JARS should only be set if Sqoop is auto-shipped
LOG.debug(Sqoop.LIB_JARS + "=" + conf.get(Sqoop.LIB_JARS));
- //copy these (which have now been localized) jars to sqoop/lib
- String destDir = conf.get(AppConfig.SQOOP_HOME_PATH) + File.separator + "lib";
String[] files = conf.getStrings(Sqoop.LIB_JARS);
+ StringBuilder jdbcJars = new StringBuilder();
for(String f : files) {
- Files.copy(new File(f), new File(destDir + File.separator + f));
+ jdbcJars.append(f).append(File.pathSeparator);
+ }
+ jdbcJars.setLength(jdbcJars.length() - 1);
+ //this makes the jars available to Sqoop client
+ if(TempletonUtils.isset(System.getenv("HADOOP_CLASSPATH"))) {
+ env.put("HADOOP_CLASSPATH", System.getenv("HADOOP_CLASSPATH") + File.pathSeparator + jdbcJars.toString());
+ }
+ else {
+ env.put("HADOOP_CLASSPATH", jdbcJars.toString());
}
}
}