You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ss...@apache.org on 2016/11/18 23:23:26 UTC
hive git commit: HIVE-15219. LLAP: Allow additional slider global
parameters to be set while creating the LLAP package. (Siddharth Seth,
reviewed by Gopal V)
Repository: hive
Updated Branches:
refs/heads/master bbcff5ea1 -> 678e6cac0
HIVE-15219. LLAP: Allow additional slider global parameters to be set while creating the LLAP package. (Siddharth Seth, reviewed by Gopal V)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/678e6cac
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/678e6cac
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/678e6cac
Branch: refs/heads/master
Commit: 678e6cac091931314f9afabe97f32c63f9036e0f
Parents: bbcff5e
Author: Siddharth Seth <ss...@apache.org>
Authored: Fri Nov 18 15:24:39 2016 -0800
Committer: Siddharth Seth <ss...@apache.org>
Committed: Fri Nov 18 15:24:39 2016 -0800
----------------------------------------------------------------------
.../hadoop/hive/llap/cli/LlapOptionsProcessor.java | 6 ++++++
llap-server/src/main/resources/package.py | 16 ++++++++++++++++
llap-server/src/main/resources/templates.py | 3 ++-
3 files changed, 24 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/678e6cac/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java
index 8b98aee..e680dfc 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java
@@ -52,6 +52,7 @@ public class LlapOptionsProcessor {
public static final String OPTION_JAVA_HOME = "javaHome"; // forward via config.json
public static final String OPTION_HIVECONF = "hiveconf"; // llap-daemon-site if relevant parameter
public static final String OPTION_SLIDER_AM_CONTAINER_MB = "slider-am-container-mb"; // forward as arg
+ public static final String OPTION_SLIDER_APPCONFIG_GLOBAL = "slider-appconfig-global"; // forward as arg
public static final String OPTION_LLAP_QUEUE = "queue"; // forward via config.json
public static final String OPTION_IO_THREADS = "iothreads"; // llap-daemon-site
@@ -256,6 +257,11 @@ public class LlapOptionsProcessor {
.withLongOpt(OPTION_SLIDER_AM_CONTAINER_MB)
.withDescription("The size of the slider AppMaster container in MB").create('b'));
+ options.addOption(OptionBuilder.withValueSeparator().hasArgs(2).withArgName("property=value")
+ .withLongOpt(OPTION_SLIDER_APPCONFIG_GLOBAL)
+ .withDescription("Property (key=value) to be set in the global section of the Slider appConfig")
+ .create());
+
options.addOption(OptionBuilder.hasArg().withArgName(OPTION_IO_THREADS)
.withLongOpt(OPTION_IO_THREADS).withDescription("executor per instance").create('t'));
http://git-wip-us.apache.org/repos/asf/hive/blob/678e6cac/llap-server/src/main/resources/package.py
----------------------------------------------------------------------
diff --git a/llap-server/src/main/resources/package.py b/llap-server/src/main/resources/package.py
index 9d61058..380c6a8 100644
--- a/llap-server/src/main/resources/package.py
+++ b/llap-server/src/main/resources/package.py
@@ -3,6 +3,7 @@
import sys,os,stat
import argparse
from json import loads as json_parse
+from json import dumps as json_print
from os.path import exists, join, relpath
from time import gmtime, strftime
import shutil
@@ -57,6 +58,19 @@ def zipdir(path, zip, prefix="."):
src = join(root, file)
dst = src.replace(path, prefix)
zip.write(src, dst)
+
+def slider_appconfig_global_property(arg):
+ kv = arg.split("=")
+ if len(kv) != 2:
+ raise argparse.ArgumentTypeError("Value must be split into two parts separated by =")
+ return tuple(kv)
+
+def construct_slider_site_global_string(kvs):
+ if not kvs:
+ return ""
+ kvs = map(lambda a : a[0], kvs)
+ return ",\n" + ",\n".join([" %s:%s" % (json_print(k), json_print(v)) for (k,v) in kvs])
+
def main(args):
version = os.getenv("HIVE_VERSION")
@@ -74,6 +88,7 @@ def main(args):
parser.add_argument("--logger", default="RFA")
parser.add_argument("--chaosmonkey", type=int, default=0)
parser.add_argument("--slider-am-container-mb", type=int, default=1024)
+ parser.add_argument("--slider-appconfig-global", nargs='*', type=slider_appconfig_global_property, action='append')
parser.add_argument("--slider-keytab-dir", default="")
parser.add_argument("--slider-keytab", default="")
parser.add_argument("--slider-principal", default="")
@@ -128,6 +143,7 @@ def main(args):
"monkey_percentage" : monkey_percentage,
"monkey_enabled" : args.chaosmonkey > 0,
"slider.am.container.mb" : args.slider_am_container_mb,
+ "slider_appconfig_global_append": construct_slider_site_global_string(args.slider_appconfig_global),
"slider_am_jvm_heapsize" : slider_am_jvm_heapsize,
"slider_keytab_dir" : slider_keytab_dir,
"slider_keytab" : slider_keytab,
http://git-wip-us.apache.org/repos/asf/hive/blob/678e6cac/llap-server/src/main/resources/templates.py
----------------------------------------------------------------------
diff --git a/llap-server/src/main/resources/templates.py b/llap-server/src/main/resources/templates.py
index 5684f33..36162bb 100644
--- a/llap-server/src/main/resources/templates.py
+++ b/llap-server/src/main/resources/templates.py
@@ -84,7 +84,8 @@ appConfig = """
"internal.chaos.monkey.probability.amlaunchfailure": "0",
"internal.chaos.monkey.probability.containerfailure": "%(monkey_percentage)d",
"internal.chaos.monkey.interval.seconds": "%(monkey_interval)d",
- "internal.chaos.monkey.enabled": "%(monkey_enabled)s"
+ "internal.chaos.monkey.enabled": "%(monkey_enabled)s"%(slider_appconfig_global_append)s
+
},
"components": {
"slider-appmaster": {