You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by th...@apache.org on 2014/04/02 23:18:08 UTC
svn commit: r1584196 - in
/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton:
HiveDelegator.java tool/JobSubmissionConstants.java tool/LaunchMapper.java
Author: thejas
Date: Wed Apr 2 21:18:08 2014
New Revision: 1584196
URL: http://svn.apache.org/r1584196
Log:
HIVE-6780 : Set tez credential file property along with MR conf property for Tez jobs (Eugene Koifman via Thejas Nair)
Modified:
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java?rev=1584196&r1=1584195&r2=1584196&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java Wed Apr 2 21:18:08 2014
@@ -27,6 +27,7 @@ import java.util.List;
import java.util.Map;
import org.apache.commons.exec.ExecuteException;
+import org.apache.hive.hcatalog.templeton.tool.JobSubmissionConstants;
import org.apache.hive.hcatalog.templeton.tool.TempletonControllerJob;
import org.apache.hive.hcatalog.templeton.tool.TempletonUtils;
@@ -76,6 +77,11 @@ public class HiveDelegator extends Launc
args.add("--hiveconf");
args.add(TempletonControllerJob.TOKEN_FILE_ARG_PLACEHOLDER);
+ //this is needed specifcally for Hive on Tez (in addition to
+ //JobSubmissionConstants.TOKEN_FILE_ARG_PLACEHOLDER)
+ args.add("--hiveconf");
+ args.add(JobSubmissionConstants.TOKEN_FILE_ARG_PLACEHOLDER_TEZ);
+
for (String prop : appConf.getStrings(AppConfig.HIVE_PROPS_NAME)) {
args.add("--hiveconf");
args.add(TempletonUtils.quoteForWindows(prop));
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java?rev=1584196&r1=1584195&r2=1584196&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java Wed Apr 2 21:18:08 2014
@@ -41,6 +41,8 @@ public interface JobSubmissionConstants
*/
public static final String TOKEN_FILE_ARG_PLACEHOLDER =
"__MR_JOB_CREDENTIALS_OPTION=WEBHCAT_TOKEN_FILE_LOCATION__";
+ public static final String TOKEN_FILE_ARG_PLACEHOLDER_TEZ =
+ "__TEZ_CREDENTIALS_OPTION=WEBHCAT_TOKEN_FILE_LOCATION_TEZ__";
/**
* constants needed for Pig job submission
*/
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java?rev=1584196&r1=1584195&r2=1584196&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java Wed Apr 2 21:18:08 2014
@@ -100,41 +100,47 @@ public class LaunchMapper extends Mapper
Map<String, String> env = TempletonUtils.hadoopUserEnv(user, overrideClasspath);
handlePigEnvVars(conf, env);
List<String> jarArgsList = new LinkedList<String>(Arrays.asList(jarArgs));
- String tokenFile = System.getenv("HADOOP_TOKEN_FILE_LOCATION");
-
+ handleTokenFile(jarArgsList, JobSubmissionConstants.TOKEN_FILE_ARG_PLACEHOLDER, "mapreduce.job.credentials.binary");
+ handleTokenFile(jarArgsList, JobSubmissionConstants.TOKEN_FILE_ARG_PLACEHOLDER_TEZ, "tez.credentials.path");
+ boolean overrideLog4jProps = conf.get(OVERRIDE_CONTAINER_LOG4J_PROPS) == null ?
+ false : Boolean.valueOf(conf.get(OVERRIDE_CONTAINER_LOG4J_PROPS));
+ return TrivialExecService.getInstance().run(jarArgsList, removeEnv, env, overrideLog4jProps);
+ }
+ /**
+ * Replace placeholder with actual "prop=file". This is done multiple times (possibly) since
+ * Tez and MR use different property names
+ */
+ private static void handleTokenFile(List<String> jarArgsList, String tokenPlaceHolder, String tokenProperty) throws IOException {
+ String tokenFile = System.getenv("HADOOP_TOKEN_FILE_LOCATION");
if (tokenFile != null) {
//Token is available, so replace the placeholder
tokenFile = tokenFile.replaceAll("\"", "");
- String tokenArg = "mapreduce.job.credentials.binary=" + tokenFile;
+ String tokenArg = tokenProperty + "=" + tokenFile;
if (Shell.WINDOWS) {
try {
tokenArg = TempletonUtils.quoteForWindows(tokenArg);
} catch (BadParam e) {
- String msg = "cannot pass " + tokenFile + " to mapreduce.job.credentials.binary";
+ String msg = "cannot pass " + tokenFile + " to " + tokenProperty;
LOG.error(msg, e);
throw new IOException(msg, e);
}
}
for(int i=0; i<jarArgsList.size(); i++){
String newArg =
- jarArgsList.get(i).replace(TOKEN_FILE_ARG_PLACEHOLDER, tokenArg);
+ jarArgsList.get(i).replace(tokenPlaceHolder, tokenArg);
jarArgsList.set(i, newArg);
}
-
}else{
//No token, so remove the placeholder arg
Iterator<String> it = jarArgsList.iterator();
while(it.hasNext()){
String arg = it.next();
- if(arg.contains(TOKEN_FILE_ARG_PLACEHOLDER)){
+ if(arg.contains(tokenPlaceHolder)){
it.remove();
}
}
}
- boolean overrideLog4jProps = conf.get(OVERRIDE_CONTAINER_LOG4J_PROPS) == null ?
- false : Boolean.valueOf(conf.get(OVERRIDE_CONTAINER_LOG4J_PROPS));
- return TrivialExecService.getInstance().run(jarArgsList, removeEnv, env, overrideLog4jProps);
}
private void copyLocal(String var, Configuration conf) throws IOException {