You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by gu...@apache.org on 2014/10/18 02:18:39 UTC
svn commit: r1632705 - in /hive/branches/branch-0.14:
common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
Author: gunther
Date: Sat Oct 18 00:18:38 2014
New Revision: 1632705
URL: http://svn.apache.org/r1632705
Log:
HIVE-8452: Cleanup handling of resource configuration for tez (Gunther Hagleitner, reviewed by Vikram Dixit K)
Modified:
hive/branches/branch-0.14/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
Modified: hive/branches/branch-0.14/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1632705&r1=1632704&r2=1632705&view=diff
==============================================================================
--- hive/branches/branch-0.14/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/branches/branch-0.14/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Sat Oct 18 00:18:38 2014
@@ -958,6 +958,9 @@ public class HiveConf extends Configurat
HIVETEZCONTAINERSIZE("hive.tez.container.size", -1,
"By default Tez will spawn containers of the size of a mapper. This can be used to overwrite."),
+ HIVETEZCPUVCORES("hive.tez.cpu.vcores", -1,
+ "By default Tez will ask for however many cpus map-reduce is configured to use per container. "
+ +"This can be used to overwrite."),
HIVETEZJAVAOPTS("hive.tez.java.opts", null,
"By default Tez will use the Java options from map tasks. This can be used to overwrite."),
HIVETEZLOGLEVEL("hive.tez.log.level", "INFO",
Modified: hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java?rev=1632705&r1=1632704&r2=1632705&view=diff
==============================================================================
--- hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java (original)
+++ hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java Sat Oct 18 00:18:38 2014
@@ -431,8 +431,9 @@ public class DagUtils {
int memory = HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVETEZCONTAINERSIZE) > 0 ?
HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVETEZCONTAINERSIZE) :
conf.getInt(MRJobConfig.MAP_MEMORY_MB, MRJobConfig.DEFAULT_MAP_MEMORY_MB);
- int cpus = conf.getInt(MRJobConfig.MAP_CPU_VCORES,
- MRJobConfig.DEFAULT_MAP_CPU_VCORES);
+ int cpus = HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVETEZCPUVCORES) > 0 ?
+ HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVETEZCPUVCORES) :
+ conf.getInt(MRJobConfig.MAP_CPU_VCORES, MRJobConfig.DEFAULT_MAP_CPU_VCORES);
return Resource.newInstance(memory, cpus);
}
@@ -452,17 +453,29 @@ public class DagUtils {
*/
private String getContainerJavaOpts(Configuration conf) {
String javaOpts = HiveConf.getVar(conf, HiveConf.ConfVars.HIVETEZJAVAOPTS);
- if (javaOpts != null && !javaOpts.isEmpty()) {
- String logLevel = HiveConf.getVar(conf, HiveConf.ConfVars.HIVETEZLOGLEVEL);
- List<String> logProps = Lists.newArrayList();
- TezUtils.addLog4jSystemProperties(logLevel, logProps);
- StringBuilder sb = new StringBuilder();
- for (String str : logProps) {
- sb.append(str).append(" ");
+
+ String logLevel = HiveConf.getVar(conf, HiveConf.ConfVars.HIVETEZLOGLEVEL);
+ List<String> logProps = Lists.newArrayList();
+ TezUtils.addLog4jSystemProperties(logLevel, logProps);
+ StringBuilder sb = new StringBuilder();
+ for (String str : logProps) {
+ sb.append(str).append(" ");
+ }
+ logLevel = sb.toString();
+
+ if (HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVETEZCONTAINERSIZE) > 0) {
+ if (javaOpts != null) {
+ return javaOpts + " " + logLevel;
+ } else {
+ return logLevel;
+ }
+ } else {
+ if (javaOpts != null && !javaOpts.isEmpty()) {
+ LOG.warn(HiveConf.ConfVars.HIVETEZJAVAOPTS + " will be ignored because "
+ + HiveConf.ConfVars.HIVETEZCONTAINERSIZE + " is not set!");
}
- return javaOpts + " " + sb.toString();
+ return logLevel + " " + MRHelpers.getJavaOptsForMRMapper(conf);
}
- return MRHelpers.getJavaOptsForMRMapper(conf);
}
private Vertex createVertex(JobConf conf, MergeJoinWork mergeJoinWork, LocalResource appJarLr,