You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jd...@apache.org on 2016/05/06 17:18:58 UTC
[08/13] hive git commit: HIVE-13656 : need to set direct memory limit
higher in LlapServiceDriver for certain edge case configurations (Sergey
Shelukhin, reviewed by Vikram Dixit K and Siddharth Seth)
HIVE-13656 : need to set direct memory limit higher in LlapServiceDriver for certain edge case configurations (Sergey Shelukhin, reviewed by Vikram Dixit K and Siddharth Seth)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3517a99e
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3517a99e
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3517a99e
Branch: refs/heads/llap
Commit: 3517a99edde061596d62b41339bacb5aac0e8290
Parents: eb2c54b
Author: Sergey Shelukhin <se...@apache.org>
Authored: Thu May 5 17:01:47 2016 -0700
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Thu May 5 17:02:36 2016 -0700
----------------------------------------------------------------------
.../hadoop/hive/llap/cli/LlapServiceDriver.java | 21 +++++++++++---------
llap-server/src/main/resources/package.py | 6 +++++-
2 files changed, 17 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/3517a99e/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
index de6d9b8..006f70f 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
@@ -236,20 +236,22 @@ public class LlapServiceDriver {
String.valueOf(options.getIoThreads()));
}
+ long cache = -1, xmx = -1;
if (options.getCache() != -1) {
- conf.set(HiveConf.ConfVars.LLAP_IO_MEMORY_MAX_SIZE.varname,
- Long.toString(options.getCache()));
+ cache = options.getCache();
+ conf.set(HiveConf.ConfVars.LLAP_IO_MEMORY_MAX_SIZE.varname, Long.toString(cache));
propsDirectOptions.setProperty(HiveConf.ConfVars.LLAP_IO_MEMORY_MAX_SIZE.varname,
- Long.toString(options.getCache()));
+ Long.toString(cache));
}
if (options.getXmx() != -1) {
// Needs more explanation here
- // Xmx is not the max heap value in JDK8
- // You need to subtract 50% of the survivor fraction from this, to get actual usable memory before it goes into GC
- long xmx = (long) (options.getXmx() / (1024 * 1024));
+ // Xmx is not the max heap value in JDK8. You need to subtract 50% of the survivor fraction
+ // from this, to get actual usable memory before it goes into GC
+ xmx = (long) (options.getXmx() / (1024 * 1024));
conf.setLong(ConfVars.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB.varname, xmx);
- propsDirectOptions.setProperty(ConfVars.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB.varname, String.valueOf(xmx));
+ propsDirectOptions.setProperty(ConfVars.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB.varname,
+ String.valueOf(xmx));
}
if (options.getLlapQueueName() != null && !options.getLlapQueueName().isEmpty()) {
@@ -258,8 +260,6 @@ public class LlapServiceDriver {
.setProperty(ConfVars.LLAP_DAEMON_QUEUE_NAME.varname, options.getLlapQueueName());
}
-
-
URL logger = conf.getResource(LlapDaemon.LOG4j2_PROPERTIES_FILE);
if (null == logger) {
@@ -460,6 +460,9 @@ public class LlapServiceDriver {
configs.put(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
conf.getInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, -1));
+ long maxDirect = (xmx > 0 && cache > 0 && xmx < cache * 1.25) ? (long)(cache * 1.25) : -1;
+ configs.put("max_direct_memory", Long.toString(maxDirect));
+
FSDataOutputStream os = lfs.create(new Path(tmpDir, "config.json"));
OutputStreamWriter w = new OutputStreamWriter(os);
configs.write(w);
http://git-wip-us.apache.org/repos/asf/hive/blob/3517a99e/llap-server/src/main/resources/package.py
----------------------------------------------------------------------
diff --git a/llap-server/src/main/resources/package.py b/llap-server/src/main/resources/package.py
index 63c0ef1..94c9d1a 100644
--- a/llap-server/src/main/resources/package.py
+++ b/llap-server/src/main/resources/package.py
@@ -101,6 +101,10 @@ def main(args):
return
config = json_parse(open(join(input, "config.json")).read())
java_home = config["java.home"]
+ max_direct_memory = config["max_direct_memory"]
+ daemon_args = args.args
+ if max_direct_memory > 0:
+ daemon_args = " -XX:MaxDirectMemorySize=%s %s" % (max_direct_memory, daemon_args)
resource = LlapResource(config)
# 5% container failure every monkey_interval seconds
monkey_percentage = 5 # 5%
@@ -114,7 +118,7 @@ def main(args):
"hadoop_home" : os.getenv("HADOOP_HOME"),
"java_home" : java_home,
"name" : resource.clusterName,
- "daemon_args" : args.args,
+ "daemon_args" : daemon_args,
"daemon_loglevel" : args.loglevel,
"queue.string" : resource.queueString,
"monkey_interval" : args.chaosmonkey,