You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by th...@apache.org on 2013/09/17 23:58:18 UTC
svn commit: r1524234 - in /hive/trunk/hcatalog:
src/test/e2e/templeton/tests/jobsubmission.conf
webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java
webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java
Author: thejas
Date: Tue Sep 17 21:58:17 2013
New Revision: 1524234
URL: http://svn.apache.org/r1524234
Log:
HIVE-4444: [HCatalog] WebHCat Hive should support equivalent parameters as Pig (Daniel Dai via Thejas Nair)
Modified:
hive/trunk/hcatalog/src/test/e2e/templeton/tests/jobsubmission.conf
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java
Modified: hive/trunk/hcatalog/src/test/e2e/templeton/tests/jobsubmission.conf
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/src/test/e2e/templeton/tests/jobsubmission.conf?rev=1524234&r1=1524233&r2=1524234&view=diff
==============================================================================
--- hive/trunk/hcatalog/src/test/e2e/templeton/tests/jobsubmission.conf (original)
+++ hive/trunk/hcatalog/src/test/e2e/templeton/tests/jobsubmission.conf Tue Sep 17 21:58:17 2013
@@ -377,7 +377,37 @@ $cfg =
},
-
+ {
+ # test to use "add jar" command in hive script, the jar must be shipped to
+ # templeton controller job using "files". Test #9 is a positive test case
+ # for this case when jar is shipped, #10 is a negative test case
+ 'num' => 9,
+ 'method' => 'POST',
+ 'url' => ':TEMPLETON_URL:/templeton/v1/hive',
+ 'post_options' => ['user.name=:UNAME:','execute=add jar piggybank.jar', 'files=:INPDIR_HDFS:/piggybank.jar',],
+ 'json_field_substr_match' => { 'id' => '\d+'},
+ #results
+ 'status_code' => 200,
+ 'check_job_created' => 1,
+ 'check_job_complete' => 'SUCCESS',
+ 'check_job_exit_value' => 0,
+ 'check_call_back' => 1,
+ },
+ {
+ # negative test case for "add jar" when jar is not shipped to templeton
+ # controller job using "files"
+ 'num' => 10,
+ 'method' => 'POST',
+ 'url' => ':TEMPLETON_URL:/templeton/v1/hive',
+ 'post_options' => ['user.name=:UNAME:','execute=add jar piggybank.jar',],
+ 'json_field_substr_match' => { 'id' => '\d+'},
+ #results
+ 'status_code' => 200,
+ 'check_job_created' => 1,
+ 'check_job_complete' => 'SUCCESS',
+ 'check_job_exit_value' => 1,
+ 'check_call_back' => 1,
+ },
]
},
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java?rev=1524234&r1=1524233&r2=1524234&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java Tue Sep 17 21:58:17 2013
@@ -22,6 +22,7 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URISyntaxException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
import org.apache.commons.exec.ExecuteException;
@@ -41,24 +42,26 @@ public class HiveDelegator extends Launc
public EnqueueBean run(String user,
String execute, String srcFile, List<String> defines,
+ List<String> hiveArgs, String otherFiles,
String statusdir, String callback, String completedUrl)
throws NotAuthorizedException, BadParam, BusyException, QueueException,
ExecuteException, IOException, InterruptedException
{
runAs = user;
- List<String> args = makeArgs(execute, srcFile, defines, statusdir,
+ List<String> args = makeArgs(execute, srcFile, defines, hiveArgs, otherFiles, statusdir,
completedUrl);
return enqueueController(user, callback, args);
}
private List<String> makeArgs(String execute, String srcFile,
- List<String> defines, String statusdir, String completedUrl)
+ List<String> defines, List<String> hiveArgs, String otherFiles,
+ String statusdir, String completedUrl)
throws BadParam, IOException, InterruptedException
{
ArrayList<String> args = new ArrayList<String>();
try {
- args.addAll(makeBasicArgs(execute, srcFile, statusdir, completedUrl));
+ args.addAll(makeBasicArgs(execute, srcFile, otherFiles, statusdir, completedUrl));
args.add("--");
args.add(appConf.hivePath());
@@ -77,6 +80,7 @@ public class HiveDelegator extends Launc
args.add("--hiveconf");
args.add(prop);
}
+ args.addAll(hiveArgs);
if (TempletonUtils.isset(execute)) {
args.add("-e");
args.add(execute);
@@ -94,8 +98,8 @@ public class HiveDelegator extends Launc
return args;
}
- private List<String> makeBasicArgs(String execute, String srcFile,
- String statusdir, String completedUrl)
+ private List<String> makeBasicArgs(String execute, String srcFile, String otherFiles,
+ String statusdir, String completedUrl)
throws URISyntaxException, FileNotFoundException, IOException,
InterruptedException
{
@@ -106,6 +110,11 @@ public class HiveDelegator extends Launc
allFiles.add(TempletonUtils.hadoopFsFilename(srcFile, appConf,
runAs));
+ if (TempletonUtils.isset(otherFiles)) {
+ String[] ofs = TempletonUtils.hadoopFsListAsArray(otherFiles, appConf, runAs);
+ allFiles.addAll(Arrays.asList(ofs));
+ }
+
args.addAll(makeLauncherArgs(appConf, statusdir, completedUrl, allFiles));
args.add("-archives");
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java?rev=1524234&r1=1524233&r2=1524234&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java Tue Sep 17 21:58:17 2013
@@ -659,12 +659,25 @@ public class Server {
/**
* Run a Hive job.
+ * @param execute SQL statement to run, equivalent to "-e" from hive command line
+ * @param srcFile name of hive script file to run, equivalent to "-f" from hive
+ * command line
+ * @param hiveArgs additional command line argument passed to the hive command line.
+ * Please check https://cwiki.apache.org/Hive/languagemanual-cli.html
+ * for detailed explanation of command line arguments
+ * @param otherFiles additional files to be shipped to the launcher, such as the jars
+ * used in "add jar" statement in hive script
+ * @param defines shortcut for command line arguments "--define"
+ * @param statusdir where the stderr/stdout of templeton controller job goes
+ * @param callback callback url when the hive job finishes
*/
@POST
@Path("hive")
@Produces({MediaType.APPLICATION_JSON})
public EnqueueBean hive(@FormParam("execute") String execute,
@FormParam("file") String srcFile,
+ @FormParam("arg") List<String> hiveArgs,
+ @FormParam("files") String otherFiles,
@FormParam("define") List<String> defines,
@FormParam("statusdir") String statusdir,
@FormParam("callback") String callback)
@@ -675,7 +688,7 @@ public class Server {
throw new BadParam("Either execute or file parameter required");
HiveDelegator d = new HiveDelegator(appConf);
- return d.run(getDoAsUser(), execute, srcFile, defines,
+ return d.run(getDoAsUser(), execute, srcFile, defines, hiveArgs, otherFiles,
statusdir, callback, getCompletedUrl());
}