You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by we...@apache.org on 2017/05/17 02:52:41 UTC
[03/50] [abbrv] hive git commit: HIVE-16639. LLAP: Derive shuffle
thread counts and keep-alive connections from instance count. (Siddharth Seth,
reviewed by Gopal V)
HIVE-16639. LLAP: Derive shuffle thread counts and keep-alive connections from instance count. (Siddharth Seth, reviewed by Gopal V)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3baaca74
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3baaca74
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3baaca74
Branch: refs/heads/hive-14535
Commit: 3baaca747d7bd81d58a8651074af5aed67067c6f
Parents: 113a099
Author: Siddharth Seth <ss...@HW10890.local>
Authored: Fri May 12 15:49:27 2017 -0700
Committer: Siddharth Seth <ss...@HW10890.local>
Committed: Fri May 12 15:49:27 2017 -0700
----------------------------------------------------------------------
.../apache/hadoop/hive/llap/shufflehandler/ShuffleHandler.java | 4 ++--
llap-server/src/main/resources/package.py | 6 +++++-
2 files changed, 7 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/3baaca74/llap-server/src/java/org/apache/hadoop/hive/llap/shufflehandler/ShuffleHandler.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/shufflehandler/ShuffleHandler.java b/llap-server/src/java/org/apache/hadoop/hive/llap/shufflehandler/ShuffleHandler.java
index 085c977..0705225 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/shufflehandler/ShuffleHandler.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/shufflehandler/ShuffleHandler.java
@@ -171,7 +171,7 @@ public class ShuffleHandler implements AttemptRegistrationListener {
public static final String SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED =
"llap.shuffle.connection-keep-alive.enable";
- public static final boolean DEFAULT_SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED = false;
+ public static final boolean DEFAULT_SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED = true;
public static final String SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT =
"llap.shuffle.connection-keep-alive.timeout";
@@ -194,7 +194,7 @@ public class ShuffleHandler implements AttemptRegistrationListener {
public static final String MAX_SHUFFLE_THREADS = "llap.shuffle.max.threads";
// 0 implies Netty default of 2 * number of available processors
- public static final int DEFAULT_MAX_SHUFFLE_THREADS = 0;
+ public static final int DEFAULT_MAX_SHUFFLE_THREADS = Runtime.getRuntime().availableProcessors() * 3;
public static final String SHUFFLE_BUFFER_SIZE =
"llap.shuffle.transfer.buffer.size";
http://git-wip-us.apache.org/repos/asf/hive/blob/3baaca74/llap-server/src/main/resources/package.py
----------------------------------------------------------------------
diff --git a/llap-server/src/main/resources/package.py b/llap-server/src/main/resources/package.py
index 8a378ef..e83d3b0 100644
--- a/llap-server/src/main/resources/package.py
+++ b/llap-server/src/main/resources/package.py
@@ -20,6 +20,7 @@ class LlapResource(object):
# convert to Mb
self.cache = config["hive.llap.io.memory.size"] / (1024*1024.0)
self.direct = config["hive.llap.io.allocator.direct"]
+ self.executors = config["hive.llap.daemon.num.executors"]
self.min_cores = -1
# compute heap + cache as final Xmx
h = self.memory
@@ -129,10 +130,13 @@ def main(args):
config = json_parse(open(join(input, "config.json")).read())
java_home = config["java.home"]
max_direct_memory = config["max_direct_memory"]
+
+ resource = LlapResource(config)
+
daemon_args = args.args
if long(max_direct_memory) > 0:
daemon_args = " -XX:MaxDirectMemorySize=%s %s" % (max_direct_memory, daemon_args)
- resource = LlapResource(config)
+ daemon_args = " -Dhttp.maxConnections=%s %s" % ((max(args.instances, resource.executors) + 1), daemon_args)
# 5% container failure every monkey_interval seconds
monkey_percentage = 5 # 5%
vars = {