You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hcatalog-commits@incubator.apache.org by tr...@apache.org on 2013/01/17 18:19:42 UTC
svn commit: r1434842 - in /incubator/hcatalog/trunk: ./
src/docs/src/documentation/content/xdocs/ webhcat/svr/src/main/config/
webhcat/svr/src/main/java/org/apache/hcatalog/templeton/
Author: travis
Date: Thu Jan 17 18:19:42 2013
New Revision: 1434842
URL: http://svn.apache.org/viewvc?rev=1434842&view=rev
Log:
HCATALOG-587 webhcat controller map task takes too much memory
Modified:
incubator/hcatalog/trunk/CHANGES.txt
incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/configuration.xml
incubator/hcatalog/trunk/webhcat/svr/src/main/config/webhcat-default.xml
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/AppConfig.java
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/LauncherDelegator.java
Modified: incubator/hcatalog/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/CHANGES.txt?rev=1434842&r1=1434841&r2=1434842&view=diff
==============================================================================
--- incubator/hcatalog/trunk/CHANGES.txt (original)
+++ incubator/hcatalog/trunk/CHANGES.txt Thu Jan 17 18:19:42 2013
@@ -176,6 +176,8 @@ Release 0.5.0 - Unreleased
OPTIMIZATIONS
BUG FIXES
+ HCAT-587 webhcat controller map task takes too much memory (thejas via traviscrawford)
+
HCAT-594 updated documentation, default.xml, job submission fix (thejas via gates)
HCAT-588 Webhcat server wouldn't start as it is unable to write templeton.log (thejas via traviscrawford)
Modified: incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/configuration.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/configuration.xml?rev=1434842&r1=1434841&r2=1434842&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/configuration.xml (original)
+++ incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/configuration.xml Thu Jan 17 18:19:42 2013
@@ -191,6 +191,12 @@ Using localhost in metastore uri does no
</tr>
<tr>
+ <td><strong>templeton.controller.mr.child.opts</strong></td>
+ <td><code>-server -Xmx256m -Djava.net.preferIPv4Stack=true</code></td>
+ <td>Java options to be passed to templeton controller map task.</td>
+ </tr>
+
+ <tr>
<td><strong>templeton.exec.envs</strong></td>
<td><code>HADOOP_PREFIX,HADOOP_HOME,JAVA_HOME,HIVE_HOME</code></td>
<td>The environment variables passed through to exec.</td>
Modified: incubator/hcatalog/trunk/webhcat/svr/src/main/config/webhcat-default.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/config/webhcat-default.xml?rev=1434842&r1=1434841&r2=1434842&view=diff
==============================================================================
--- incubator/hcatalog/trunk/webhcat/svr/src/main/config/webhcat-default.xml (original)
+++ incubator/hcatalog/trunk/webhcat/svr/src/main/config/webhcat-default.xml Thu Jan 17 18:19:42 2013
@@ -145,6 +145,21 @@
</property>
<property>
+ <name>templeton.controller.mr.child.opts</name>
+ <value>-server -Xmx256m -Djava.net.preferIPv4Stack=true</value>
+ <description>Java options to be passed to templeton controller map task.
+ The default value of mapreduce child "-Xmx" (heap memory limit)
+ might be close to what is allowed for a map task.
+ Even if templeton controller map task does not need much
+ memory, the jvm (with -server option?)
+ allocates the max memory when it starts. This along with the
+ memory used by pig/hive client it starts can end up exceeding
+ the max memory configured to be allowed for a map task
+ Use this option to set -Xmx to lower value
+ </description>
+ </property>
+
+ <property>
<name>templeton.exec.envs</name>
<value>HADOOP_PREFIX,HADOOP_HOME,JAVA_HOME,HIVE_HOME</value>
<description>The environment variables passed through to exec.</description>
Modified: incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/AppConfig.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/AppConfig.java?rev=1434842&r1=1434841&r2=1434842&view=diff
==============================================================================
--- incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/AppConfig.java (original)
+++ incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/AppConfig.java Thu Jan 17 18:19:42 2013
@@ -21,6 +21,7 @@ package org.apache.hcatalog.templeton;
import java.io.File;
import java.net.URL;
import java.util.Map;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -95,6 +96,9 @@ public class AppConfig extends Configura
public static final String TEMPLETON_JAR_NAME = "templeton.jar";
public static final String OVERRIDE_JARS_NAME = "templeton.override.jars";
public static final String OVERRIDE_JARS_ENABLED = "templeton.override.enabled";
+ public static final String TEMPLETON_CONTROLLER_MR_CHILD_OPTS
+ = "templeton.controller.mr.child.opts";
+
public static final String KERBEROS_SECRET = "templeton.kerberos.secret";
public static final String KERBEROS_PRINCIPAL = "templeton.kerberos.principal";
public static final String KERBEROS_KEYTAB = "templeton.kerberos.keytab";
@@ -103,12 +107,16 @@ public class AppConfig extends Configura
= "templeton.callback.retry.interval";
public static final String CALLBACK_RETRY_NAME
= "templeton.callback.retry.attempts";
+
+ //Hadoop property names (set by templeton logic)
public static final String HADOOP_END_INTERVAL_NAME = "job.end.retry.interval";
public static final String HADOOP_END_RETRY_NAME = "job.end.retry.attempts";
public static final String HADOOP_END_URL_NAME = "job.end.notification.url";
public static final String HADOOP_SPECULATIVE_NAME
= "mapred.map.tasks.speculative.execution";
+ public static final String HADOOP_CHILD_JAVA_OPTS = "mapred.child.java.opts";
+
private static final Log LOG = LogFactory.getLog(AppConfig.class);
public AppConfig() {
@@ -177,7 +185,12 @@ public class AppConfig extends Configura
public String kerberosSecret() { return get(KERBEROS_SECRET); }
public String kerberosPrincipal(){ return get(KERBEROS_PRINCIPAL); }
public String kerberosKeytab() { return get(KERBEROS_KEYTAB); }
+ public String controllerMRChildOpts() {
+ return get(TEMPLETON_CONTROLLER_MR_CHILD_OPTS);
+ }
+
+
public String[] overrideJars() {
if (getBoolean(OVERRIDE_JARS_ENABLED, true))
return getStrings(OVERRIDE_JARS_NAME);
Modified: incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/LauncherDelegator.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/LauncherDelegator.java?rev=1434842&r1=1434841&r2=1434842&view=diff
==============================================================================
--- incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/LauncherDelegator.java (original)
+++ incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/LauncherDelegator.java Thu Jan 17 18:19:42 2013
@@ -116,6 +116,7 @@ public class LauncherDelegator extends T
// Hadoop vars
addDef(args, "user.name", runAs);
addDef(args, AppConfig.HADOOP_SPECULATIVE_NAME, "false");
+ addDef(args, AppConfig.HADOOP_CHILD_JAVA_OPTS, appConf.controllerMRChildOpts());
// Internal vars
addDef(args, TempletonControllerJob.STATUSDIR_NAME, statusdir);