You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2014/04/21 23:53:22 UTC
svn commit: r1588991 - in /hive/trunk/hcatalog/webhcat/svr/src: main/config/
main/java/org/apache/hive/hcatalog/templeton/
main/java/org/apache/hive/hcatalog/templeton/tool/
test/java/org/apache/hive/hcatalog/templeton/tool/
Author: hashutosh
Date: Mon Apr 21 21:53:22 2014
New Revision: 1588991
URL: http://svn.apache.org/r1588991
Log:
HIVE-4576 : templeton.hive.properties does not allow values with commas (Eugene Koifman via Ashutosh Chauhan)
Modified:
hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java
hive/trunk/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml?rev=1588991&r1=1588990&r2=1588991&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml Mon Apr 21 21:53:22 2014
@@ -137,7 +137,8 @@
<property>
<name>templeton.hive.properties</name>
<value>hive.metastore.uris=thrift://localhost:9933,hive.metastore.sasl.enabled=false</value>
- <description>Properties to set when running hive.</description>
+ <description>Properties to set when running hive. This is expected to be a comma-separated prop=value list.
+ If some value is itself a comma-separated list the escape character is '\'</description>
</property>
<property>
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java?rev=1588991&r1=1588990&r2=1588991&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java Mon Apr 21 21:53:22 2014
@@ -21,6 +21,8 @@ package org.apache.hive.hcatalog.templet
import java.io.File;
import java.net.URL;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.Iterator;
@@ -31,8 +33,10 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.VersionInfo;
import org.apache.hive.hcatalog.templeton.tool.JobState;
+import org.apache.hive.hcatalog.templeton.tool.TempletonUtils;
import org.apache.hive.hcatalog.templeton.tool.ZooKeeperCleanup;
import org.apache.hive.hcatalog.templeton.tool.ZooKeeperStorage;
@@ -104,6 +108,11 @@ public class AppConfig extends Configura
* see webhcat-default.xml
*/
public static final String HCAT_HOME_PATH = "templeton.hcat.home";
+ /**
+ * is a comma separated list of name=value pairs;
+ * In case some value is itself a comma-separated list, the comma needs to
+ * be escaped with {@link org.apache.hadoop.util.StringUtils#ESCAPE_CHAR}
+ */
public static final String HIVE_PROPS_NAME = "templeton.hive.properties";
public static final String LIB_JARS_NAME = "templeton.libjars";
public static final String PIG_ARCHIVE_NAME = "templeton.pig.archive";
@@ -247,8 +256,18 @@ public class AppConfig extends Configura
public String controllerMRChildOpts() {
return get(TEMPLETON_CONTROLLER_MR_CHILD_OPTS);
}
-
-
+ /**
+ * @see #HIVE_PROPS_NAME
+ */
+ public Collection<String> hiveProps() {
+ String[] props= StringUtils.split(get(HIVE_PROPS_NAME));
+ //since raw data was (possibly) escaped to make split work,
+ //now need to remove escape chars so they don't interfere with downstream processing
+ for(int i = 0; i < props.length; i++) {
+ props[i] = TempletonUtils.unEscape(props[i]);
+ }
+ return Arrays.asList(props);
+ }
public String[] overrideJars() {
if (getBoolean(OVERRIDE_JARS_ENABLED, true))
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java?rev=1588991&r1=1588990&r2=1588991&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java Mon Apr 21 21:53:22 2014
@@ -82,7 +82,7 @@ public class HiveDelegator extends Launc
args.add("--hiveconf");
args.add(JobSubmissionConstants.TOKEN_FILE_ARG_PLACEHOLDER_TEZ);
- for (String prop : appConf.getStrings(AppConfig.HIVE_PROPS_NAME)) {
+ for (String prop : appConf.hiveProps()) {
args.add("--hiveconf");
args.add(TempletonUtils.quoteForWindows(prop));
}
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java?rev=1588991&r1=1588990&r2=1588991&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java Mon Apr 21 21:53:22 2014
@@ -45,6 +45,7 @@ public interface JobSubmissionConstants
"__TEZ_CREDENTIALS_OPTION=WEBHCAT_TOKEN_FILE_LOCATION_TEZ__";
/**
* constants needed for Pig job submission
+ * The string values here are what Pig expects to see in it's environment
*/
public static interface PigConstants {
public static final String HIVE_HOME = "HIVE_HOME";
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java?rev=1588991&r1=1588990&r2=1588991&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java Mon Apr 21 21:53:22 2014
@@ -29,6 +29,7 @@ import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.util.StringUtils;
import org.apache.hive.hcatalog.templeton.BadParam;
import org.apache.hive.hcatalog.templeton.LauncherDelegator;
@@ -79,8 +80,8 @@ public class LaunchMapper extends Mapper
}
if(conf.get(PigConstants.PIG_OPTS) != null) {
StringBuilder pigOpts = new StringBuilder();
- for(String prop : conf.get(PigConstants.PIG_OPTS).split(",")) {
- pigOpts.append("-D").append(prop).append(" ");
+ for(String prop : StringUtils.split(conf.get(PigConstants.PIG_OPTS))) {
+ pigOpts.append("-D").append(TempletonUtils.unEscape(prop)).append(" ");
}
env.put(PigConstants.PIG_OPTS, pigOpts.toString());
}
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java?rev=1588991&r1=1588990&r2=1588991&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java Mon Apr 21 21:53:22 2014
@@ -36,6 +36,7 @@ import java.util.regex.Pattern;
import javax.ws.rs.core.UriBuilder;
+import org.apache.commons.lang.StringEscapeUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -346,4 +347,8 @@ public class TempletonUtils {
args.add("call");
}
}
+
+ public static String unEscape(String input) {
+ return StringEscapeUtils.unescapeJava(input);
+ }
}
Modified: hive/trunk/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java?rev=1588991&r1=1588990&r2=1588991&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java Mon Apr 21 21:53:22 2014
@@ -290,5 +290,23 @@ public class TestTempletonUtils {
} catch (URISyntaxException ex) {
}
}
-
+
+ @Test
+ public void testPropertiesParsing() throws Exception {
+ String[] props = {"hive.metastore.uris=thrift://localhost:9933\\,thrift://127.0.0.1:9933",
+ "hive.metastore.sasl.enabled=false",
+ "hive.some.fake.path=C:\\foo\\bar.txt\\"};
+ StringBuilder input = new StringBuilder();
+ for(String prop : props) {
+ if(input.length() > 0) {
+ input.append(',');
+ }
+ input.append(prop);
+ }
+ String[] newProps = StringUtils.split(input.toString());
+ for(int i = 0; i < newProps.length; i++) {
+ Assert.assertEquals("Pre/post split values don't match",
+ TempletonUtils.unEscape(props[i]), TempletonUtils.unEscape(newProps[i]));
+ }
+ }
}