You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by th...@apache.org on 2014/06/06 23:07:26 UTC
svn commit: r1601012 - in /hive/trunk/hcatalog:
src/test/e2e/templeton/deployers/config/hive/ webhcat/svr/src/main/config/
webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/
webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/ ...
Author: thejas
Date: Fri Jun 6 21:07:26 2014
New Revision: 1601012
URL: http://svn.apache.org/r1601012
Log:
HIVE-7065 : Hive jobs in webhcat run in default mr mode even in Hive on Tez setup (Eugene Koifman via Thejas Nair)
Modified:
hive/trunk/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-site.xml
hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java
hive/trunk/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java
Modified: hive/trunk/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-site.xml
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-site.xml?rev=1601012&r1=1601011&r2=1601012&view=diff
==============================================================================
--- hive/trunk/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-site.xml (original)
+++ hive/trunk/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-site.xml Fri Jun 6 21:07:26 2014
@@ -31,6 +31,10 @@
<value>thrift://localhost:9933</value>
<description>For Hive CLI to connect to</description>
</property>
+ <property>
+ <name>hive.execution.engine</name>
+ <value>mr</value><!--tez-->
+ </property>
<!--
This enables client side Hive (RDBMS style) security
<property>
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml?rev=1601012&r1=1601011&r2=1601012&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml Fri Jun 6 21:07:26 2014
@@ -131,8 +131,9 @@
<property>
<name>templeton.hive.properties</name>
<value>hive.metastore.uris=thrift://localhost:9933,hive.metastore.sasl.enabled=false</value>
- <description>Properties to set when running hive. This is expected to be a comma-separated prop=value list.
- If some value is itself a comma-separated list the escape character is '\'</description>
+ <description>Properties to set when running hive (during job sumission). This is expected to
+ be a comma-separated prop=value list. If some value is itself a comma-separated list the
+ escape character is '\'</description>
</property>
<property>
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java?rev=1601012&r1=1601011&r2=1601012&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java Fri Jun 6 21:07:26 2014
@@ -33,10 +33,10 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.VersionInfo;
import org.apache.hive.hcatalog.templeton.tool.JobState;
-import org.apache.hive.hcatalog.templeton.tool.TempletonUtils;
import org.apache.hive.hcatalog.templeton.tool.ZooKeeperCleanup;
import org.apache.hive.hcatalog.templeton.tool.ZooKeeperStorage;
@@ -111,7 +111,8 @@ public class AppConfig extends Configura
/**
* is a comma separated list of name=value pairs;
* In case some value is itself a comma-separated list, the comma needs to
- * be escaped with {@link org.apache.hadoop.util.StringUtils#ESCAPE_CHAR}
+ * be escaped with {@link org.apache.hadoop.util.StringUtils#ESCAPE_CHAR}. See other usage
+ * of escape/unescape methods in {@link org.apache.hadoop.util.StringUtils} in webhcat.
*/
public static final String HIVE_PROPS_NAME = "templeton.hive.properties";
public static final String SQOOP_ARCHIVE_NAME = "templeton.sqoop.archive";
@@ -167,8 +168,49 @@ public class AppConfig extends Configura
loadOneFileConfig(hadoopConfDir, fname);
}
ProxyUserSupport.processProxyuserConfig(this);
+ handleHiveProperties();
LOG.info(dumpEnvironent());
}
+ /**
+ * When auto-shipping hive tar (for example when hive query or pig script
+ * is submitted via webhcat), Hive client is launched on some remote node where Hive has not
+ * been installed. We need pass some properties to that client to make sure it connects to the
+ * right Metastore, configures Tez, etc. Here we look for such properties in hive config,
+ * and set a comma-separated list of key values in {@link #HIVE_PROPS_NAME}.
+ * Note that the user may choose to set the same keys in HIVE_PROPS_NAME directly, in which case
+ * those values should take precedence.
+ */
+ private void handleHiveProperties() {
+ HiveConf hiveConf = new HiveConf();//load hive-site.xml from classpath
+ List<String> interestingPropNames = Arrays.asList(
+ "hive.metastore.uris","hive.metastore.sasl.enabled",
+ "hive.metastore.execute.setugi","hive.execution.engine");
+
+ //each items is a "key=value" format
+ List<String> webhcatHiveProps = new ArrayList<String>(hiveProps());
+ for(String interestingPropName : interestingPropNames) {
+ String value = hiveConf.get(interestingPropName);
+ if(value != null) {
+ boolean found = false;
+ for(String whProp : webhcatHiveProps) {
+ if(whProp.startsWith(interestingPropName + "=")) {
+ found = true;
+ break;
+ }
+ }
+ if(!found) {
+ webhcatHiveProps.add(interestingPropName + "=" + value);
+ }
+ }
+ }
+ StringBuilder hiveProps = new StringBuilder();
+ for(String whProp : webhcatHiveProps) {
+ //make sure to escape separator char in prop values
+ hiveProps.append(hiveProps.length() > 0 ? "," : "").append(StringUtils.escapeString(whProp));
+ }
+ set(HIVE_PROPS_NAME, hiveProps.toString());
+ }
+
private static void logConfigLoadAttempt(String path) {
LOG.info("Attempting to load config file: " + path);
}
@@ -198,7 +240,8 @@ public class AppConfig extends Configura
}
});
for(Map.Entry<String, String> entry : configVals) {
- sb.append(entry.getKey()).append('=').append(entry.getValue()).append('\n');
+ //use get() to make sure variable substitution works
+ sb.append(entry.getKey()).append('=').append(get(entry.getKey())).append('\n');
}
return sb.toString();
}
@@ -266,7 +309,7 @@ public class AppConfig extends Configura
//since raw data was (possibly) escaped to make split work,
//now need to remove escape chars so they don't interfere with downstream processing
for(int i = 0; i < props.length; i++) {
- props[i] = TempletonUtils.unEscape(props[i]);
+ props[i] = StringUtils.unEscapeString(props[i]);
}
return Arrays.asList(props);
}
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java?rev=1601012&r1=1601011&r2=1601012&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java Fri Jun 6 21:07:26 2014
@@ -81,7 +81,7 @@ public class LaunchMapper extends Mapper
if(conf.get(PigConstants.PIG_OPTS) != null) {
StringBuilder pigOpts = new StringBuilder();
for(String prop : StringUtils.split(conf.get(PigConstants.PIG_OPTS))) {
- pigOpts.append("-D").append(TempletonUtils.unEscape(prop)).append(" ");
+ pigOpts.append("-D").append(StringUtils.unEscapeString(prop)).append(" ");
}
env.put(PigConstants.PIG_OPTS, pigOpts.toString());
}
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java?rev=1601012&r1=1601011&r2=1601012&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java Fri Jun 6 21:07:26 2014
@@ -347,8 +347,4 @@ public class TempletonUtils {
args.add("call");
}
}
-
- public static String unEscape(String input) {
- return StringEscapeUtils.unescapeJava(input);
- }
}
Modified: hive/trunk/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java?rev=1601012&r1=1601011&r2=1601012&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java Fri Jun 6 21:07:26 2014
@@ -306,7 +306,7 @@ public class TestTempletonUtils {
String[] newProps = StringUtils.split(input.toString());
for(int i = 0; i < newProps.length; i++) {
Assert.assertEquals("Pre/post split values don't match",
- TempletonUtils.unEscape(props[i]), TempletonUtils.unEscape(newProps[i]));
+ StringUtils.unEscapeString(props[i]), StringUtils.unEscapeString(newProps[i]));
}
}
}