You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by th...@apache.org on 2013/10/01 02:50:59 UTC
svn commit: r1527836 - in
/hive/branches/branch-0.12/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton:
HiveDelegator.java JarDelegator.java PigDelegator.java Server.java
StreamingDelegator.java tool/TempletonUtils.java
Author: thejas
Date: Tue Oct 1 00:50:59 2013
New Revision: 1527836
URL: http://svn.apache.org/r1527836
Log:
HIVE-5035: [WebHCat] Hardening parameters for Windows (Daniel Dai via Thejas Nair)
Modified:
hive/branches/branch-0.12/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java
hive/branches/branch-0.12/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JarDelegator.java
hive/branches/branch-0.12/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/PigDelegator.java
hive/branches/branch-0.12/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java
hive/branches/branch-0.12/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/StreamingDelegator.java
hive/branches/branch-0.12/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java
Modified: hive/branches/branch-0.12/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java?rev=1527836&r1=1527835&r2=1527836&view=diff
==============================================================================
--- hive/branches/branch-0.12/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java (original)
+++ hive/branches/branch-0.12/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java Tue Oct 1 00:50:59 2013
@@ -75,16 +75,18 @@ public class HiveDelegator extends Launc
for (String prop : appConf.getStrings(AppConfig.HIVE_PROPS_NAME)) {
args.add("--hiveconf");
- args.add(prop);
+ args.add(TempletonUtils.quoteForWindows(prop));
}
for (String prop : defines) {
args.add("--hiveconf");
- args.add(prop);
+ args.add(TempletonUtils.quoteForWindows(prop));
+ }
+ for (String hiveArg : hiveArgs) {
+ args.add(TempletonUtils.quoteForWindows(hiveArg));
}
- args.addAll(hiveArgs);
if (TempletonUtils.isset(execute)) {
args.add("-e");
- args.add(execute);
+ args.add(TempletonUtils.quoteForWindows(execute));
} else if (TempletonUtils.isset(srcFile)) {
args.add("-f");
args.add(TempletonUtils.hadoopFsPath(srcFile, appConf, runAs)
Modified: hive/branches/branch-0.12/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JarDelegator.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JarDelegator.java?rev=1527836&r1=1527835&r2=1527836&view=diff
==============================================================================
--- hive/branches/branch-0.12/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JarDelegator.java (original)
+++ hive/branches/branch-0.12/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JarDelegator.java Tue Oct 1 00:50:59 2013
@@ -74,22 +74,27 @@ public class JarDelegator extends Launch
if (TempletonUtils.isset(mainClass))
args.add(mainClass);
if (TempletonUtils.isset(libjars)) {
+ String libjarsListAsString =
+ TempletonUtils.hadoopFsListAsString(libjars, appConf, runAs);
args.add("-libjars");
- args.add(TempletonUtils.hadoopFsListAsString(libjars, appConf,
- runAs));
+ args.add(TempletonUtils.quoteForWindows(libjarsListAsString));
}
if (TempletonUtils.isset(files)) {
+ String filesListAsString =
+ TempletonUtils.hadoopFsListAsString(files, appConf, runAs);
args.add("-files");
- args.add(TempletonUtils.hadoopFsListAsString(files, appConf,
- runAs));
+ args.add(TempletonUtils.quoteForWindows(filesListAsString));
}
//the token file location comes after mainClass, as a -Dprop=val
args.add("-D" + TempletonControllerJob.TOKEN_FILE_ARG_PLACEHOLDER);
- for (String d : defines)
- args.add("-D" + d);
-
- args.addAll(jarArgs);
+ for (String d : defines) {
+ args.add("-D");
+ TempletonUtils.quoteForWindows(d);
+ }
+ for (String arg : jarArgs) {
+ args.add(TempletonUtils.quoteForWindows(arg));
+ }
} catch (FileNotFoundException e) {
throw new BadParam(e.getMessage());
} catch (URISyntaxException e) {
Modified: hive/branches/branch-0.12/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/PigDelegator.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/PigDelegator.java?rev=1527836&r1=1527835&r2=1527836&view=diff
==============================================================================
--- hive/branches/branch-0.12/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/PigDelegator.java (original)
+++ hive/branches/branch-0.12/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/PigDelegator.java Tue Oct 1 00:50:59 2013
@@ -78,10 +78,12 @@ public class PigDelegator extends Launch
//the token file location should be first argument of pig
args.add("-D" + TempletonControllerJob.TOKEN_FILE_ARG_PLACEHOLDER);
- args.addAll(pigArgs);
+ for (String pigArg : pigArgs) {
+ args.add(TempletonUtils.quoteForWindows(pigArg));
+ }
if (TempletonUtils.isset(execute)) {
args.add("-execute");
- args.add(execute);
+ args.add(TempletonUtils.quoteForWindows(execute));
} else if (TempletonUtils.isset(srcFile)) {
args.add("-file");
args.add(TempletonUtils.hadoopFsPath(srcFile, appConf, runAs)
Modified: hive/branches/branch-0.12/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java?rev=1527836&r1=1527835&r2=1527836&view=diff
==============================================================================
--- hive/branches/branch-0.12/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java (original)
+++ hive/branches/branch-0.12/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java Tue Oct 1 00:50:59 2013
@@ -587,7 +587,9 @@ public class Server {
@FormParam("output") String output,
@FormParam("mapper") String mapper,
@FormParam("reducer") String reducer,
- @FormParam("file") List<String> files,
+ @FormParam("combiner") String combiner,
+ @FormParam("file") List<String> fileList,
+ @FormParam("files") String files,
@FormParam("define") List<String> defines,
@FormParam("cmdenv") List<String> cmdenvs,
@FormParam("arg") List<String> args,
@@ -607,6 +609,8 @@ public class Server {
userArgs.put("output", output);
userArgs.put("mapper", mapper);
userArgs.put("reducer", reducer);
+ userArgs.put("combiner", combiner);
+ userArgs.put("file", fileList);
userArgs.put("files", files);
userArgs.put("define", defines);
userArgs.put("cmdenv", cmdenvs);
@@ -617,8 +621,8 @@ public class Server {
checkEnableLogPrerequisite(enablelog, statusdir);
StreamingDelegator d = new StreamingDelegator(appConf);
- return d.run(getDoAsUser(), userArgs, inputs, output, mapper, reducer,
- files, defines, cmdenvs, args,
+ return d.run(getDoAsUser(), userArgs, inputs, output, mapper, reducer, combiner,
+ fileList, files, defines, cmdenvs, args,
statusdir, callback, getCompletedUrl(), enablelog, JobType.STREAMING);
}
Modified: hive/branches/branch-0.12/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/StreamingDelegator.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/StreamingDelegator.java?rev=1527836&r1=1527835&r2=1527836&view=diff
==============================================================================
--- hive/branches/branch-0.12/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/StreamingDelegator.java (original)
+++ hive/branches/branch-0.12/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/StreamingDelegator.java Tue Oct 1 00:50:59 2013
@@ -24,6 +24,7 @@ import java.util.List;
import java.util.Map;
import org.apache.commons.exec.ExecuteException;
+import org.apache.hive.hcatalog.templeton.tool.TempletonUtils;
/**
* Submit a streaming job to the MapReduce queue. Really just a front
@@ -38,8 +39,9 @@ public class StreamingDelegator extends
public EnqueueBean run(String user, Map<String, Object> userArgs,
List<String> inputs, String output,
- String mapper, String reducer,
- List<String> files, List<String> defines,
+ String mapper, String reducer, String combiner,
+ List<String> fileList,
+ String files, List<String> defines,
List<String> cmdenvs,
List<String> jarArgs,
String statusdir,
@@ -49,13 +51,13 @@ public class StreamingDelegator extends
JobType jobType)
throws NotAuthorizedException, BadParam, BusyException, QueueException,
ExecuteException, IOException, InterruptedException {
- List<String> args = makeArgs(inputs, output, mapper, reducer,
- files, defines, cmdenvs, jarArgs);
+ List<String> args = makeArgs(inputs, output, mapper, reducer, combiner,
+ fileList, cmdenvs, jarArgs);
JarDelegator d = new JarDelegator(appConf);
return d.run(user, userArgs,
appConf.streamingJar(), null,
- null, null, args, defines,
+ null, files, args, defines,
statusdir, callback, completedUrl, enableLog, jobType);
}
@@ -63,10 +65,12 @@ public class StreamingDelegator extends
String output,
String mapper,
String reducer,
- List<String> files,
- List<String> defines,
+ String combiner,
+ List<String> fileList,
List<String> cmdenvs,
- List<String> jarArgs) {
+ List<String> jarArgs)
+ throws BadParam
+ {
ArrayList<String> args = new ArrayList<String>();
for (String input : inputs) {
args.add("-input");
@@ -79,13 +83,24 @@ public class StreamingDelegator extends
args.add("-reducer");
args.add(reducer);
- for (String f : files)
- args.add("-file" + f);
- for (String d : defines)
- args.add("-D" + d);
- for (String e : cmdenvs)
- args.add("-cmdenv" + e);
- args.addAll(jarArgs);
+ if (TempletonUtils.isset(combiner)) {
+ args.add("-combiner");
+ args.add(combiner);
+ }
+
+ for (String f : fileList) {
+ args.add("-file");
+ args.add(f);
+ }
+
+ for (String e : cmdenvs) {
+ args.add("-cmdenv");
+ args.add(TempletonUtils.quoteForWindows(e));
+ }
+
+ for (String arg : jarArgs) {
+ args.add(TempletonUtils.quoteForWindows(arg));
+ }
return args;
}
Modified: hive/branches/branch-0.12/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java?rev=1527836&r1=1527835&r2=1527836&view=diff
==============================================================================
--- hive/branches/branch-0.12/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java (original)
+++ hive/branches/branch-0.12/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java Tue Oct 1 00:50:59 2013
@@ -26,6 +26,7 @@ import java.net.URISyntaxException;
import java.net.URL;
import java.net.URLConnection;
import java.security.PrivilegedExceptionAction;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
@@ -39,8 +40,10 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.StringUtils;
import org.apache.hive.hcatalog.templeton.UgiFactory;
+import org.apache.hive.hcatalog.templeton.BadParam;
/**
* General utility methods.
@@ -296,4 +299,46 @@ public class TempletonUtils {
return env;
}
+
+ // Add double quotes around the given input parameter if it is not already
+ // quoted. Quotes are not allowed in the middle of the parameter, and
+ // BadParam exception is thrown if this is the case.
+ //
+ // This method should be used to escape parameters before they get passed to
+ // Windows cmd scripts (specifically, special characters like a comma or an
+ // equal sign might be lost as part of the cmd script processing if not
+ // under quotes).
+ public static String quoteForWindows(String param) throws BadParam {
+ if (Shell.WINDOWS) {
+ if (param != null && param.length() > 0) {
+ String nonQuotedPart = param;
+ boolean addQuotes = true;
+ if (param.charAt(0) == '\"' && param.charAt(param.length() - 1) == '\"') {
+ if (param.length() < 2)
+ throw new BadParam("Passed in parameter is incorrectly quoted: " + param);
+
+ addQuotes = false;
+ nonQuotedPart = param.substring(1, param.length() - 1);
+ }
+
+ // If we have any quotes other then the outside quotes, throw
+ if (nonQuotedPart.contains("\"")) {
+ throw new BadParam("Passed in parameter is incorrectly quoted: " + param);
+ }
+
+ if (addQuotes) {
+ param = '\"' + param + '\"';
+ }
+ }
+ }
+ return param;
+ }
+
+ public static void addCmdForWindows(ArrayList<String> args) {
+ if(Shell.WINDOWS){
+ args.add("cmd");
+ args.add("/c");
+ args.add("call");
+ }
+ }
}