svn commit: r1494383 - in /hive/trunk/hcatalog/webhcat/svr/src/main: config/webhcat-default.xml java/org/apache/hcatalog/templeton/AppConfig.java java/org/apache/hcatalog/templeton/LauncherDelegator.java

[ga...@apache.org]

Author: gates
Date: Tue Jun 18 23:45:08 2013
New Revision: 1494383

URL: http://svn.apache.org/r1494383
Log:
HIVE-4679 WebHCat can deadlock Hadoop if the number of concurrently running tasks if higher or equal than the number of mappers (ekoifman via gates)

Modified:
    hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml
    hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/AppConfig.java
    hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/LauncherDelegator.java

Modified: hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml?rev=1494383&r1=1494382&r2=1494383&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml Tue Jun 18 23:45:08 2013
@@ -225,4 +225,14 @@
     <description>The maximum age of a templeton job</description>
   </property>
 
+  <property>
+    <name>templeton.hadoop.queue.name</name>
+    <value>default</value>
+    <description>
+    MapReduce queue name where WebHCat map-only jobs will be
+    submitted to. Can be used to avoid a deadlock where all map slots
+    in the cluster are taken over by Templeton launcher tasks.
+    </description>
+  </property>
+
 </configuration>

Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/AppConfig.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/AppConfig.java?rev=1494383&r1=1494382&r2=1494383&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/AppConfig.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/AppConfig.java Tue Jun 18 23:45:08 2013
@@ -83,6 +83,7 @@ public class AppConfig extends Configura
     public static final String EXEC_MAX_BYTES_NAME = "templeton.exec.max-output-bytes";
     public static final String EXEC_MAX_PROCS_NAME = "templeton.exec.max-procs";
     public static final String EXEC_TIMEOUT_NAME   = "templeton.exec.timeout";
+    public static final String HADOOP_QUEUE_NAME   = "templeton.hadoop.queue.name";
     public static final String HADOOP_NAME         = "templeton.hadoop";
     public static final String HADOOP_CONF_DIR     = "templeton.hadoop.conf.dir";
     public static final String HCAT_NAME           = "templeton.hcat";
@@ -175,6 +176,7 @@ public class AppConfig extends Configura
 
     public String templetonJar()     { return get(TEMPLETON_JAR_NAME); }
     public String libJars()          { return get(LIB_JARS_NAME); }
+    public String hadoopQueueName()  { return get(HADOOP_QUEUE_NAME); }
     public String clusterHadoop()    { return get(HADOOP_NAME); }
     public String clusterHcat()      { return get(HCAT_NAME); }
     public String pigPath()          { return get(PIG_PATH_NAME); }

Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/LauncherDelegator.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/LauncherDelegator.java?rev=1494383&r1=1494382&r2=1494383&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/LauncherDelegator.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/LauncherDelegator.java Tue Jun 18 23:45:08 2013
@@ -125,6 +125,9 @@ public class LauncherDelegator extends T
         addDef(args, TempletonControllerJob.OVERRIDE_CLASSPATH,
             makeOverrideClasspath(appConf));
 
+        // Hadoop queue information
+        addDef(args, "mapred.job.queue.name", appConf.hadoopQueueName());
+
         // Job vars
         addStorageVars(args);
         addCompletionVars(args, completedUrl);