You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ss...@apache.org on 2016/03/07 19:59:51 UTC
hive git commit: HIVE-13156. Allow specifying the name of the queue
in which llap will run, and the size of the slider AM. (Siddharth Seth,
reviewed by Gopal V)
Repository: hive
Updated Branches:
refs/heads/master 109a3a3c1 -> 4655ae8c5
HIVE-13156. Allow specifying the name of the queue in which llap will run, and the size of the slider AM. (Siddharth Seth, reviewed by Gopal V)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/4655ae8c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/4655ae8c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/4655ae8c
Branch: refs/heads/master
Commit: 4655ae8c57d5bddd189ef637737a2ea9bc937ad9
Parents: 109a3a3
Author: Siddharth Seth <ss...@apache.org>
Authored: Mon Mar 7 10:58:42 2016 -0800
Committer: Siddharth Seth <ss...@apache.org>
Committed: Mon Mar 7 10:58:42 2016 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/hive/conf/HiveConf.java | 3 +++
.../hive/llap/cli/LlapOptionsProcessor.java | 28 +++++++++++++++++---
.../hadoop/hive/llap/cli/LlapServiceDriver.java | 12 +++++++++
llap-server/src/main/resources/package.py | 11 ++++++++
llap-server/src/main/resources/templates.py | 6 +++--
5 files changed, 54 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/4655ae8c/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 2723dad..5098851 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -2581,6 +2581,9 @@ public class HiveConf extends Configuration {
"YARN shuffle port for LLAP-daemon-hosted shuffle.", "llap.daemon.yarn.shuffle.port"),
LLAP_DAEMON_YARN_CONTAINER_MB("hive.llap.daemon.yarn.container.mb", -1,
"llap server yarn container size in MB. Used in LlapServiceDriver and package.py", "llap.daemon.yarn.container.mb"),
+ LLAP_DAEMON_QUEUE_NAME("hive.llap.daemon.queue.name", null,
+ "Queue name within which the llap slider application will run." +
+ " Used in LlapServiceDriver and package.py"),
LLAP_DAEMON_SHUFFLE_DIR_WATCHER_ENABLED("hive.llap.daemon.shuffle.dir.watcher.enabled", false,
"TODO doc", "llap.daemon.shuffle.dir-watcher.enabled"),
LLAP_DAEMON_AM_LIVENESS_HEARTBEAT_INTERVAL_MS(
http://git-wip-us.apache.org/repos/asf/hive/blob/4655ae8c/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java
index 1732df8..c43bf97 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java
@@ -18,9 +18,7 @@
package org.apache.hadoop.hive.llap.cli;
-import java.io.FileInputStream;
import java.io.IOException;
-import java.io.InputStream;
import java.util.Properties;
import javax.annotation.Nonnull;
@@ -31,6 +29,7 @@ import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
+import org.apache.hadoop.hive.conf.HiveConf;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.util.StringUtils;
@@ -51,6 +50,8 @@ public class LlapOptionsProcessor {
public static final String OPTION_AUXHBASE = "auxhbase"; // used to localize jars
public static final String OPTION_JAVA_HOME = "javaHome"; // forward via config.json
public static final String OPTION_HIVECONF = "hiveconf"; // llap-daemon-site if relevant parameter
+ public static final String OPTION_SLIDER_AM_CONTAINER_MB = "slider-am-container-mb"; // forward as arg
+ public static final String OPTION_LLAP_QUEUE = "queue"; // forward via config.json
public class LlapOptions {
private final int instances;
@@ -64,10 +65,11 @@ public class LlapOptionsProcessor {
private final boolean isHbase;
private final Properties conf;
private final String javaPath;
+ private final String llapQueueName;
public LlapOptions(String name, int instances, String directory, int executors, long cache,
long size, long xmx, String jars, boolean isHbase,
- @Nonnull Properties hiveconf, String javaPath)
+ @Nonnull Properties hiveconf, String javaPath, String llapQueueName)
throws ParseException {
if (instances <= 0) {
throw new ParseException("Invalid configuration: " + instances
@@ -84,6 +86,7 @@ public class LlapOptionsProcessor {
this.isHbase = isHbase;
this.conf = hiveconf;
this.javaPath = javaPath;
+ this.llapQueueName = llapQueueName;
}
public String getName() {
@@ -129,6 +132,10 @@ public class LlapOptionsProcessor {
public String getJavaPath() {
return javaPath;
}
+
+ public String getLlapQueueName() {
+ return llapQueueName;
+ }
}
protected static final Logger l4j = LoggerFactory.getLogger(LlapOptionsProcessor.class.getName());
@@ -169,6 +176,10 @@ public class LlapOptionsProcessor {
options.addOption(OptionBuilder.hasArg().withArgName(OPTION_XMX).withLongOpt(OPTION_XMX)
.withDescription("working memory size").create('w'));
+ options.addOption(OptionBuilder.hasArg().withArgName(OPTION_LLAP_QUEUE)
+ .withLongOpt(OPTION_LLAP_QUEUE)
+ .withDescription("The queue within which LLAP will be started").create('q'));
+
options.addOption(OptionBuilder.hasArg().withArgName(OPTION_AUXJARS).withLongOpt(OPTION_AUXJARS)
.withDescription("additional jars to package (by default, JSON SerDe jar is packaged"
+ " if available)").create('j'));
@@ -187,6 +198,10 @@ public class LlapOptionsProcessor {
.withDescription("Use value for given property. Overridden by explicit parameters")
.create());
+ options.addOption(OptionBuilder.hasArg().withArgName(OPTION_SLIDER_AM_CONTAINER_MB)
+ .withLongOpt(OPTION_SLIDER_AM_CONTAINER_MB)
+ .withDescription("The size of the slider AppMaster container in MB").create());
+
// [-H|--help]
options.addOption(new Option("H", "help", false, "Print help information"));
}
@@ -215,6 +230,9 @@ public class LlapOptionsProcessor {
final long xmx = parseSuffixed(commandLine.getOptionValue(OPTION_XMX, "-1"));
final boolean isHbase = Boolean.parseBoolean(commandLine.getOptionValue(OPTION_AUXHBASE, "true"));
+ final String queueName = commandLine.getOptionValue(OPTION_LLAP_QUEUE,
+ HiveConf.ConfVars.LLAP_DAEMON_QUEUE_NAME.getDefaultValue());
+
final Properties hiveconf;
if (commandLine.hasOption(OPTION_HIVECONF)) {
@@ -227,10 +245,12 @@ public class LlapOptionsProcessor {
if (commandLine.hasOption(OPTION_JAVA_HOME)) {
javaHome = commandLine.getOptionValue(OPTION_JAVA_HOME);
}
+
// loglevel, chaosmonkey & args are parsed by the python processor
return new LlapOptions(
- name, instances, directory, executors, cache, size, xmx, jars, isHbase, hiveconf, javaHome);
+ name, instances, directory, executors, cache, size, xmx, jars, isHbase, hiveconf, javaHome,
+ queueName);
}
private void printUsage() {
http://git-wip-us.apache.org/repos/asf/hive/blob/4655ae8c/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
index 4cce53b..f6ca79d 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
@@ -224,6 +224,12 @@ public class LlapServiceDriver {
propsDirectOptions.setProperty(ConfVars.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB.varname, String.valueOf(xmx));
}
+ if (options.getLlapQueueName() != null && !options.getLlapQueueName().isEmpty()) {
+ conf.set(ConfVars.LLAP_DAEMON_QUEUE_NAME.varname, options.getLlapQueueName());
+ propsDirectOptions
+ .setProperty(ConfVars.LLAP_DAEMON_QUEUE_NAME.varname, options.getLlapQueueName());
+ }
+
URL logger = conf.getResource(LlapDaemon.LOG4j2_PROPERTIES_FILE);
@@ -381,6 +387,12 @@ public class LlapServiceDriver {
configs.put(ConfVars.LLAP_DAEMON_NUM_EXECUTORS.varname, HiveConf.getIntVar(conf,
ConfVars.LLAP_DAEMON_NUM_EXECUTORS));
+ // Let YARN pick the queue name, if it isn't provided in hive-site, or via the command-line
+ if (HiveConf.getVar(conf, ConfVars.LLAP_DAEMON_QUEUE_NAME) != null) {
+ configs.put(ConfVars.LLAP_DAEMON_QUEUE_NAME.varname,
+ HiveConf.getVar(conf, ConfVars.LLAP_DAEMON_QUEUE_NAME));
+ }
+
configs.put(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
conf.getInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, -1));
http://git-wip-us.apache.org/repos/asf/hive/blob/4655ae8c/llap-server/src/main/resources/package.py
----------------------------------------------------------------------
diff --git a/llap-server/src/main/resources/package.py b/llap-server/src/main/resources/package.py
index 0204b7b..58c43be 100644
--- a/llap-server/src/main/resources/package.py
+++ b/llap-server/src/main/resources/package.py
@@ -37,6 +37,12 @@ class LlapResource(object):
self.container_cores = self.cores
self.heap_size = h
+ if (not config.get("hive.llap.daemon.queue.name","")):
+ self.queueString = ""
+ else:
+ self.queueString = "--queue "
+ self.queueString += config["hive.llap.daemon.queue.name"]
+
def __repr__(self):
return "<LlapResource heap=%d container=%d>" % (self.heap_size, self.container_size)
@@ -61,6 +67,7 @@ def main(args):
parser.add_argument("--name", default="llap0")
parser.add_argument("--loglevel", default="INFO")
parser.add_argument("--chaosmonkey", type=int, default=0)
+ parser.add_argument("--slider-am-container-mb", type=int, default=1024)
parser.add_argument("--slider-keytab-dir", default="")
parser.add_argument("--slider-keytab", default="")
parser.add_argument("--slider-principal", default="")
@@ -71,6 +78,7 @@ def main(args):
(args, unknown_args) = parser.parse_known_args(args)
input = args.input
output = args.output
+ slider_am_jvm_heapsize = max(args.slider_am_container_mb * 0.8, args.slider_am_container_mb - 1024)
slider_keytab_dir = args.slider_keytab_dir
slider_keytab = args.slider_keytab
slider_principal = args.slider_principal
@@ -103,9 +111,12 @@ def main(args):
"name" : args.name,
"daemon_args" : args.args,
"daemon_loglevel" : args.loglevel,
+ "queue.string" : resource.queueString,
"monkey_interval" : args.chaosmonkey,
"monkey_percentage" : monkey_percentage,
"monkey_enabled" : args.chaosmonkey > 0,
+ "slider.am.container.mb" : args.slider_am_container_mb,
+ "slider_am_jvm_heapsize" : slider_am_jvm_heapsize,
"slider_keytab_dir" : slider_keytab_dir,
"slider_keytab" : slider_keytab,
"slider_principal" : slider_principal
http://git-wip-us.apache.org/repos/asf/hive/blob/4655ae8c/llap-server/src/main/resources/templates.py
----------------------------------------------------------------------
diff --git a/llap-server/src/main/resources/templates.py b/llap-server/src/main/resources/templates.py
index 277a49c..8baa927 100644
--- a/llap-server/src/main/resources/templates.py
+++ b/llap-server/src/main/resources/templates.py
@@ -88,7 +88,7 @@ appConfig = """
},
"components": {
"slider-appmaster": {
- "jvm.heapsize": "1024M",
+ "jvm.heapsize": "%(slider_am_jvm_heapsize)dM",
"slider.hdfs.keytab.dir": "%(slider_keytab_dir)s",
"slider.am.login.keytab.name": "%(slider_keytab)s",
"slider.keytab.principal.name": "%(slider_principal)s"
@@ -106,6 +106,8 @@ resources = """
},
"components": {
"slider-appmaster": {
+ "yarn.memory": "%(slider.am.container.mb)d",
+ "yarn.component.instances": "1"
},
"LLAP": {
"yarn.role.priority": "1",
@@ -126,5 +128,5 @@ BASEDIR=$(dirname $0)
slider stop %(name)s
slider destroy %(name)s --force || slider destroy %(name)s
slider install-package --name LLAP --package $BASEDIR/llap-%(version)s.zip --replacepkg
-slider create %(name)s --resources $BASEDIR/resources.json --template $BASEDIR/appConfig.json
+slider create %(name)s --resources $BASEDIR/resources.json --template $BASEDIR/appConfig.json %(queue.string)s
"""