You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by th...@apache.org on 2014/06/09 22:34:54 UTC
svn commit: r1601500 - in /hive/trunk/hcatalog/webhcat/svr/src/main: config/
java/org/apache/hive/hcatalog/templeton/
java/org/apache/hive/hcatalog/templeton/tool/
Author: thejas
Date: Mon Jun 9 20:34:54 2014
New Revision: 1601500
URL: http://svn.apache.org/r1601500
Log:
HIVE-7155 : WebHCat controller job exceeds container memory limit (shanyu zhao via Thejas Nair)
Modified:
hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/LauncherDelegator.java
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml?rev=1601500&r1=1601499&r2=1601500&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml Mon Jun 9 20:34:54 2014
@@ -187,6 +187,17 @@
</property>
<property>
+ <name>templeton.mapper.memory.mb</name>
+ <value></value>
+ <description>Templeton controller job's Launch mapper's memory limit in
+ megabytes. When submitting controller job, Templeton will overwrite
+ mapreduce.map.memory.mb with this value. If empty, Templeton will
+ not set mapreduce.map.memory.mb when submitting the controller job,
+ therefore the configuration in mapred-site.xml will be used.
+ </description>
+ </property>
+
+ <property>
<name>templeton.exec.envs</name>
<value>HADOOP_PREFIX,HADOOP_HOME,JAVA_HOME,HIVE_HOME</value>
<description>The environment variables passed through to exec.</description>
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java?rev=1601500&r1=1601499&r2=1601500&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java Mon Jun 9 20:34:54 2014
@@ -100,6 +100,7 @@ public class AppConfig extends Configura
public static final String PYTHON_NAME = "templeton.python";
public static final String HIVE_ARCHIVE_NAME = "templeton.hive.archive";
public static final String HIVE_PATH_NAME = "templeton.hive.path";
+ public static final String MAPPER_MEMORY_MB = "templeton.mapper.memory.mb";
/**
* see webhcat-default.xml
*/
@@ -142,6 +143,7 @@ public class AppConfig extends Configura
public static final String HADOOP_SPECULATIVE_NAME
= "mapred.map.tasks.speculative.execution";
public static final String HADOOP_CHILD_JAVA_OPTS = "mapred.child.java.opts";
+ public static final String HADOOP_MAP_MEMORY_MB = "mapreduce.map.memory.mb";
public static final String UNIT_TEST_MODE = "templeton.unit.test.mode";
@@ -301,6 +303,8 @@ public class AppConfig extends Configura
public String controllerMRChildOpts() {
return get(TEMPLETON_CONTROLLER_MR_CHILD_OPTS);
}
+ public String mapperMemoryMb() { return get(MAPPER_MEMORY_MB); }
+
/**
* @see #HIVE_PROPS_NAME
*/
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/LauncherDelegator.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/LauncherDelegator.java?rev=1601500&r1=1601499&r2=1601500&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/LauncherDelegator.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/LauncherDelegator.java Mon Jun 9 20:34:54 2014
@@ -103,7 +103,7 @@ public class LauncherDelegator extends T
return ugi.doAs(new PrivilegedExceptionAction<String>() {
public String run() throws Exception {
String[] array = new String[args.size()];
- TempletonControllerJob ctrl = new TempletonControllerJob(secureMeatastoreAccess);
+ TempletonControllerJob ctrl = new TempletonControllerJob(secureMeatastoreAccess, appConf);
ToolRunner.run(ctrl, args.toArray(array));
return ctrl.getSubmittedId();
}
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java?rev=1601500&r1=1601499&r2=1601500&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java Mon Jun 9 20:34:54 2014
@@ -75,14 +75,16 @@ import org.apache.thrift.TException;
public class TempletonControllerJob extends Configured implements Tool, JobSubmissionConstants {
private static final Log LOG = LogFactory.getLog(TempletonControllerJob.class);
private final boolean secureMetastoreAccess;
+ private final AppConfig appConf;
/**
* @param secureMetastoreAccess - if true, a delegation token will be created
* and added to the job
*/
- public TempletonControllerJob(boolean secureMetastoreAccess) {
+ public TempletonControllerJob(boolean secureMetastoreAccess, AppConfig conf) {
super();
this.secureMetastoreAccess = secureMetastoreAccess;
+ this.appConf = conf;
}
private JobID submittedJobId;
@@ -108,6 +110,10 @@ public class TempletonControllerJob exte
Configuration conf = getConf();
conf.set(JAR_ARGS_NAME, TempletonUtils.encodeArray(args));
+ String memoryMb = appConf.mapperMemoryMb();
+ if(memoryMb != null && memoryMb.length() != 0) {
+ conf.set(AppConfig.HADOOP_MAP_MEMORY_MB, memoryMb);
+ }
String user = UserGroupInformation.getCurrentUser().getShortUserName();
conf.set("user.name", user);
Job job = new Job(conf);