You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by th...@apache.org on 2013/11/01 23:24:01 UTC
svn commit: r1538074 - in /hive/trunk/hcatalog/webhcat/svr/src/main: config/
java/org/apache/hive/hcatalog/templeton/
java/org/apache/hive/hcatalog/templeton/tool/
Author: thejas
Date: Fri Nov 1 22:24:00 2013
New Revision: 1538074
URL: http://svn.apache.org/r1538074
Log:
HIVE-5547 : webhcat pig job submission should ship hive tar if -usehcatalog is specified (Eugene Koifman via Thejas Nair)
Modified:
hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JarDelegator.java
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/LauncherDelegator.java
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Main.java
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/PigDelegator.java
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/TempletonDelegator.java
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml?rev=1538074&r1=1538073&r2=1538074&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml Fri Nov 1 22:24:00 2013
@@ -116,6 +116,25 @@
</property>
<property>
+ <name>templeton.hive.home</name>
+ <value>hive-0.13.0-SNAPSHOT-bin.tar.gz/hive-0.13.0-SNAPSHOT-bin</value>
+ <description>
+ The path to the Hive home within the tar. This is needed if Hive is not installed on all
+ nodes in the cluster and needs to be shipped to the target node in the cluster to execute Pig
+ job which uses HCat, Hive query, etc. Has no effect if templeton.hive.archive is not set.
+ </description>
+ </property>
+ <property>
+ <name>templeton.hcat.home</name>
+ <value>hive-0.13.0-SNAPSHOT-bin.tar.gz/hive-0.13.0-SNAPSHOT-bin/hcatalog</value>
+ <description>
+ The path to the HCat home within the tar. This is needed if Hive is not installed on all
+ nodes in the cluster and needs to be shipped to the target node in the cluster to execute Pig
+ job which uses HCat, Hive query, etc. Has no effect if templeton.hive.archive is not set.
+ </description>
+ </property>
+
+ <property>
<name>templeton.hive.properties</name>
<value>hive.metastore.local=false,hive.metastore.uris=thrift://localhost:9933,hive.metastore.sasl.enabled=false</value>
<description>Properties to set when running hive.</description>
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java?rev=1538074&r1=1538073&r2=1538074&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java Fri Nov 1 22:24:00 2013
@@ -91,6 +91,14 @@ public class AppConfig extends Configura
public static final String PYTHON_NAME = "templeton.python";
public static final String HIVE_ARCHIVE_NAME = "templeton.hive.archive";
public static final String HIVE_PATH_NAME = "templeton.hive.path";
+ /**
+ * see webhcat-default.xml
+ */
+ public static final String HIVE_HOME_PATH = "templeton.hive.home";
+ /**
+ * see webhcat-default.xml
+ */
+ public static final String HCAT_HOME_PATH = "templeton.hcat.home";
public static final String HIVE_PROPS_NAME = "templeton.hive.properties";
public static final String LIB_JARS_NAME = "templeton.libjars";
public static final String PIG_ARCHIVE_NAME = "templeton.pig.archive";
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java?rev=1538074&r1=1538073&r2=1538074&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java Fri Nov 1 22:24:00 2013
@@ -128,7 +128,7 @@ public class HiveDelegator extends Launc
if (appConf.hiveArchive() != null && !appConf.hiveArchive().equals(""))
{
- args.add("-archives");
+ args.add(ARCHIVES);
args.add(appConf.hiveArchive());
}
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JarDelegator.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JarDelegator.java?rev=1538074&r1=1538073&r2=1538074&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JarDelegator.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JarDelegator.java Fri Nov 1 22:24:00 2013
@@ -43,14 +43,14 @@ public class JarDelegator extends Launch
String libjars, String files,
List<String> jarArgs, List<String> defines,
String statusdir, String callback,
- boolean usehcatalog, String completedUrl,
+ boolean usesHcatalog, String completedUrl,
boolean enablelog, JobType jobType)
throws NotAuthorizedException, BadParam, BusyException, QueueException,
ExecuteException, IOException, InterruptedException {
runAs = user;
List<String> args = makeArgs(jar, mainClass,
libjars, files, jarArgs, defines,
- statusdir, usehcatalog, completedUrl, enablelog, jobType);
+ statusdir, usesHcatalog, completedUrl, enablelog, jobType);
return enqueueController(user, userArgs, callback, args);
}
@@ -58,7 +58,7 @@ public class JarDelegator extends Launch
private List<String> makeArgs(String jar, String mainClass,
String libjars, String files,
List<String> jarArgs, List<String> defines,
- String statusdir, boolean usehcatalog, String completedUrl,
+ String statusdir, boolean usesHcatalog, String completedUrl,
boolean enablelog, JobType jobType)
throws BadParam, IOException, InterruptedException {
ArrayList<String> args = new ArrayList<String>();
@@ -72,7 +72,7 @@ public class JarDelegator extends Launch
TempletonUtils.addCmdForWindows(args);
//check if the rest command specified explicitly to use hcatalog
- if(usehcatalog){
+ if(usesHcatalog){
addHiveMetaStoreTokenArg();
}
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/LauncherDelegator.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/LauncherDelegator.java?rev=1538074&r1=1538073&r2=1538074&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/LauncherDelegator.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/LauncherDelegator.java Fri Nov 1 22:24:00 2013
@@ -97,6 +97,9 @@ public class LauncherDelegator extends T
private String queueAsUser(UserGroupInformation ugi, final List<String> args)
throws IOException, InterruptedException {
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("Launching job: " + args);
+ }
return ugi.doAs(new PrivilegedExceptionAction<String>() {
public String run() throws Exception {
String[] array = new String[args.size()];
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Main.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Main.java?rev=1538074&r1=1538073&r2=1538074&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Main.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Main.java Fri Nov 1 22:24:00 2013
@@ -206,10 +206,13 @@ public class Main {
public FilterHolder makeAuthFilter() {
FilterHolder authFilter = new FilterHolder(AuthFilter.class);
if (UserGroupInformation.isSecurityEnabled()) {
+ //http://hadoop.apache.org/docs/r1.1.1/api/org/apache/hadoop/security/authentication/server/AuthenticationFilter.html
authFilter.setInitParameter("dfs.web.authentication.signature.secret",
conf.kerberosSecret());
+ //https://svn.apache.org/repos/asf/hadoop/common/branches/branch-1.2/src/packages/templates/conf/hdfs-site.xml
authFilter.setInitParameter("dfs.web.authentication.kerberos.principal",
conf.kerberosPrincipal());
+ //http://https://svn.apache.org/repos/asf/hadoop/common/branches/branch-1.2/src/packages/templates/conf/hdfs-site.xml
authFilter.setInitParameter("dfs.web.authentication.kerberos.keytab",
conf.kerberosKeytab());
}
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/PigDelegator.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/PigDelegator.java?rev=1538074&r1=1538073&r2=1538074&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/PigDelegator.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/PigDelegator.java Fri Nov 1 22:24:00 2013
@@ -29,6 +29,7 @@ import java.util.Map;
import org.apache.commons.exec.ExecuteException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hive.hcatalog.templeton.tool.JobSubmissionConstants;
import org.apache.hive.hcatalog.templeton.tool.TempletonControllerJob;
import org.apache.hive.hcatalog.templeton.tool.TempletonUtils;
@@ -47,13 +48,13 @@ public class PigDelegator extends Launch
String execute, String srcFile,
List<String> pigArgs, String otherFiles,
String statusdir, String callback,
- boolean usehcatalog, String completedUrl, boolean enablelog)
+ boolean usesHcatalog, String completedUrl, boolean enablelog)
throws NotAuthorizedException, BadParam, BusyException, QueueException,
ExecuteException, IOException, InterruptedException {
runAs = user;
List<String> args = makeArgs(execute,
srcFile, pigArgs,
- otherFiles, statusdir, usehcatalog, completedUrl, enablelog);
+ otherFiles, statusdir, usesHcatalog, completedUrl, enablelog);
return enqueueController(user, userArgs, callback, args);
}
@@ -64,7 +65,7 @@ public class PigDelegator extends Launch
* @param pigArgs pig command line arguments
* @param otherFiles files to be copied to the map reduce cluster
* @param statusdir status dir location
- * @param usehcatalog whether the command uses hcatalog/needs to connect
+ * @param usesHcatalog whether the command uses hcatalog/needs to connect
* to hive metastore server
* @param completedUrl call back url
* @return list of arguments
@@ -74,10 +75,14 @@ public class PigDelegator extends Launch
*/
private List<String> makeArgs(String execute, String srcFile,
List<String> pigArgs, String otherFiles,
- String statusdir, boolean usehcatalog,
+ String statusdir, boolean usesHcatalog,
String completedUrl, boolean enablelog)
throws BadParam, IOException, InterruptedException {
ArrayList<String> args = new ArrayList<String>();
+ //check if the REST command specified explicitly to use hcatalog
+ // or if it says that implicitly using the pig -useHCatalog arg
+ boolean needsMetastoreAccess = usesHcatalog || hasPigArgUseHcat(pigArgs);
+
try {
ArrayList<String> allFiles = new ArrayList<String>();
if (TempletonUtils.isset(srcFile)) {
@@ -89,12 +94,32 @@ public class PigDelegator extends Launch
}
args.addAll(makeLauncherArgs(appConf, statusdir, completedUrl, allFiles, enablelog, JobType.PIG));
- if (appConf.pigArchive() != null && !appConf.pigArchive().equals(""))
- {
- args.add("-archives");
- args.add(appConf.pigArchive());
+ boolean shipPigTar = appConf.pigArchive() != null && !appConf.pigArchive().equals("");
+ boolean shipHiveTar = needsMetastoreAccess && appConf.hiveArchive() != null
+ && !appConf.hiveArchive().equals("");
+ if(shipPigTar || shipHiveTar) {
+ args.add(ARCHIVES);
+ StringBuilder archives = new StringBuilder();
+ if(shipPigTar) {
+ archives.append(appConf.pigArchive());
+ }
+ if(shipPigTar && shipHiveTar) {
+ archives.append(",");
+ }
+ if(shipHiveTar) {
+ archives.append(appConf.hiveArchive());
+ }
+ args.add(archives.toString());
+ }
+ if(shipHiveTar) {
+ addDef(args, JobSubmissionConstants.PigConstants.HIVE_HOME,
+ appConf.get(AppConfig.HIVE_HOME_PATH));
+ addDef(args, JobSubmissionConstants.PigConstants.HCAT_HOME,
+ appConf.get(AppConfig.HCAT_HOME_PATH));
+ //Pig which uses HCat will pass this to HCat so that it can find the metastore
+ addDef(args, JobSubmissionConstants.PigConstants.PIG_OPTS,
+ appConf.get(AppConfig.HIVE_PROPS_NAME));
}
-
args.add("--");
TempletonUtils.addCmdForWindows(args);
args.add(appConf.pigPath());
@@ -104,9 +129,7 @@ public class PigDelegator extends Launch
for (String pigArg : pigArgs) {
args.add(TempletonUtils.quoteForWindows(pigArg));
}
- //check if the REST command specified explicitly to use hcatalog
- // or if it says that implicitly using the pig -useHCatalog arg
- if(usehcatalog || hasPigArgUseHcat(pigArgs)){
+ if(needsMetastoreAccess) {
addHiveMetaStoreTokenArg();
}
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java?rev=1538074&r1=1538073&r2=1538074&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java Fri Nov 1 22:24:00 2013
@@ -635,7 +635,7 @@ public class Server {
/**
* Run a MapReduce Jar job.
* Params correspond to the REST api params
- * @param usehcatalog if {@code true}, means the Jar uses HCat and thus needs to access
+ * @param usesHcatalog if {@code true}, means the Jar uses HCat and thus needs to access
* metastore, which requires additional steps for WebHCat to perform in a secure cluster.
* @param callback URL which WebHCat will call when the hive job finishes
* @see org.apache.hive.hcatalog.templeton.tool.TempletonControllerJob
@@ -651,7 +651,7 @@ public class Server {
@FormParam("define") List<String> defines,
@FormParam("statusdir") String statusdir,
@FormParam("callback") String callback,
- @FormParam("usehcatalog") boolean usehcatalog,
+ @FormParam("usehcatalog") boolean usesHcatalog,
@FormParam("enablelog") boolean enablelog)
throws NotAuthorizedException, BusyException, BadParam, QueueException,
ExecuteException, IOException, InterruptedException {
@@ -677,14 +677,14 @@ public class Server {
return d.run(getDoAsUser(), userArgs,
jar, mainClass,
libjars, files, args, defines,
- statusdir, callback, usehcatalog, getCompletedUrl(), enablelog, JobType.JAR);
+ statusdir, callback, usesHcatalog, getCompletedUrl(), enablelog, JobType.JAR);
}
/**
* Run a Pig job.
- * Params correspond to the REST api params. If '-useHCatalog' is in the {@code pigArgs, usehcatalog},
+ * Params correspond to the REST api params. If '-useHCatalog' is in the {@code pigArgs, usesHcatalog},
* is interpreted as true.
- * @param usehcatalog if {@code true}, means the Pig script uses HCat and thus needs to access
+ * @param usesHcatalog if {@code true}, means the Pig script uses HCat and thus needs to access
* metastore, which requires additional steps for WebHCat to perform in a secure cluster.
* This does nothing to ensure that Pig is installed on target node in the cluster.
* @param callback URL which WebHCat will call when the hive job finishes
@@ -699,7 +699,7 @@ public class Server {
@FormParam("files") String otherFiles,
@FormParam("statusdir") String statusdir,
@FormParam("callback") String callback,
- @FormParam("usehcatalog") boolean usehcatalog,
+ @FormParam("usehcatalog") boolean usesHcatalog,
@FormParam("enablelog") boolean enablelog)
throws NotAuthorizedException, BusyException, BadParam, QueueException,
ExecuteException, IOException, InterruptedException {
@@ -725,7 +725,7 @@ public class Server {
return d.run(getDoAsUser(), userArgs,
execute, srcFile,
pigArgs, otherFiles,
- statusdir, callback, usehcatalog, getCompletedUrl(), enablelog);
+ statusdir, callback, usesHcatalog, getCompletedUrl(), enablelog);
}
/**
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/TempletonDelegator.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/TempletonDelegator.java?rev=1538074&r1=1538073&r2=1538074&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/TempletonDelegator.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/TempletonDelegator.java Fri Nov 1 22:24:00 2013
@@ -24,6 +24,11 @@ package org.apache.hive.hcatalog.templet
* or hive.
*/
public class TempletonDelegator {
+ /**
+ * http://hadoop.apache.org/docs/r1.0.4/commands_manual.html#Generic+Options
+ */
+ public static final String ARCHIVES = "-archives";
+
protected AppConfig appConf;
public TempletonDelegator(AppConfig appConf) {
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java?rev=1538074&r1=1538073&r2=1538074&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java Fri Nov 1 22:24:00 2013
@@ -34,4 +34,12 @@ public interface JobSubmissionConstants
public static final int WATCHER_TIMEOUT_SECS = 10;
public static final int KEEP_ALIVE_MSEC = 60 * 1000;
public static final String TOKEN_FILE_ARG_PLACEHOLDER = "__WEBHCAT_TOKEN_FILE_LOCATION__";
+ /**
+ * constants needed for Pig job submission
+ */
+ public static interface PigConstants {
+ public static final String HIVE_HOME = "HIVE_HOME";
+ public static final String HCAT_HOME = "HCAT_HOME";
+ public static final String PIG_OPTS = "PIG_OPTS";
+ }
}
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java?rev=1538074&r1=1538073&r2=1538074&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java Fri Nov 1 22:24:00 2013
@@ -33,6 +33,7 @@ import org.apache.hive.hcatalog.templeto
import org.apache.hive.hcatalog.templeton.LauncherDelegator;
import java.io.BufferedReader;
+import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
@@ -65,8 +66,25 @@ public class LaunchMapper extends Mapper
* it will end up in 'syslog' of this Map task. For example, look for KeepAlive heartbeat msgs.
*/
private static final Log LOG = LogFactory.getLog(LaunchMapper.class);
-
-
+ /**
+ * When a Pig job is submitted and it uses HCat, WebHCat may be configured to ship hive tar
+ * to the target node. Pig on the target node needs some env vars configured.
+ */
+ private static void handlePigEnvVars(Configuration conf, Map<String, String> env) {
+ if(conf.get(PigConstants.HIVE_HOME) != null) {
+ env.put(PigConstants.HIVE_HOME, new File(conf.get(PigConstants.HIVE_HOME)).getAbsolutePath());
+ }
+ if(conf.get(PigConstants.HCAT_HOME) != null) {
+ env.put(PigConstants.HCAT_HOME, new File(conf.get(PigConstants.HCAT_HOME)).getAbsolutePath());
+ }
+ if(conf.get(PigConstants.PIG_OPTS) != null) {
+ StringBuilder pigOpts = new StringBuilder();
+ for(String prop : conf.get(PigConstants.PIG_OPTS).split(",")) {
+ pigOpts.append("-D").append(prop).append(" ");
+ }
+ env.put(PigConstants.PIG_OPTS, pigOpts.toString());
+ }
+ }
protected Process startJob(Context context, String user, String overrideClasspath)
throws IOException, InterruptedException {
Configuration conf = context.getConfiguration();
@@ -79,8 +97,8 @@ public class LaunchMapper extends Mapper
removeEnv.add("hadoop-command");
removeEnv.add("CLASS");
removeEnv.add("mapredcommand");
- Map<String, String> env = TempletonUtils.hadoopUserEnv(user,
- overrideClasspath);
+ Map<String, String> env = TempletonUtils.hadoopUserEnv(user, overrideClasspath);
+ handlePigEnvVars(conf, env);
List<String> jarArgsList = new LinkedList<String>(Arrays.asList(jarArgs));
String tokenFile = System.getenv("HADOOP_TOKEN_FILE_LOCATION");
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java?rev=1538074&r1=1538073&r2=1538074&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java Fri Nov 1 22:24:00 2013
@@ -22,6 +22,7 @@ import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.security.PrivilegedExceptionAction;
+import java.util.Arrays;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -182,6 +183,9 @@ public class TempletonControllerJob exte
@Override
public int run(String[] args) throws IOException, InterruptedException, ClassNotFoundException,
TException {
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("Preparing to submit job: " + Arrays.toString(args));
+ }
Configuration conf = getConf();
conf.set(JAR_ARGS_NAME, TempletonUtils.encodeArray(args));