You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sw...@apache.org on 2016/07/21 21:37:44 UTC
ambari git commit: AMBARI-17818. Tez & Hive: Memory management should
use scaled down Xmx for sizing buffers. (Gopal V via Swapan Shridhar)
Repository: ambari
Updated Branches:
refs/heads/trunk a30d15738 -> 2516a059a
AMBARI-17818. Tez & Hive: Memory management should use scaled down Xmx for sizing buffers. (Gopal V via Swapan Shridhar)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/2516a059
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/2516a059
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/2516a059
Branch: refs/heads/trunk
Commit: 2516a059a7ca116d218bb6411fccd4ce11ad59ff
Parents: a30d157
Author: Swapan Shridhar <ss...@hortonworks.com>
Authored: Thu Jul 21 14:35:38 2016 -0700
Committer: Swapan Shridhar <ss...@hortonworks.com>
Committed: Thu Jul 21 14:37:36 2016 -0700
----------------------------------------------------------------------
.../resources/stacks/HDP/2.2/services/stack_advisor.py | 2 +-
.../resources/stacks/HDP/2.3/services/stack_advisor.py | 3 ++-
.../test/python/stacks/2.2/common/test_stack_advisor.py | 4 ++--
.../test/python/stacks/2.3/common/test_stack_advisor.py | 10 +++++-----
4 files changed, 10 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/2516a059/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
index 1dd7a07..b77980a 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
@@ -364,7 +364,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
putHiveSiteProperty("hive.tez.dynamic.partition.pruning", "true")
container_size = configurations["hive-site"]["properties"]["hive.tez.container.size"]
- container_size_bytes = int(container_size)*1024*1024
+ container_size_bytes = int(int(container_size)*0.8*1024*1024) # Xmx == 80% of container
# Memory
putHiveSiteProperty("hive.auto.convert.join.noconditionaltask.size", int(round(container_size_bytes/3)))
putHiveSitePropertyAttribute("hive.auto.convert.join.noconditionaltask.size", "maximum", container_size_bytes)
http://git-wip-us.apache.org/repos/asf/ambari/blob/2516a059/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
index 24047ce..8d5cdc9 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
@@ -92,7 +92,8 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
# remove 2gb limit for tez.runtime.io.sort.mb
# in HDP 2.3 "tez.runtime.sorter.class" is set by default to PIPELINED, in other case comment calculation code below
taskResourceMemory = int(configurations["tez-site"]["properties"]["tez.task.resource.memory.mb"])
- putTezProperty("tez.runtime.io.sort.mb", int(taskResourceMemory * 0.4))
+ # fit io.sort.mb into tenured regions
+ putTezProperty("tez.runtime.io.sort.mb", int(taskResourceMemory * 0.8 * 0.33))
if "tez-site" in services["configurations"] and "tez.runtime.sorter.class" in services["configurations"]["tez-site"]["properties"]:
if services["configurations"]["tez-site"]["properties"]["tez.runtime.sorter.class"] == "LEGACY":
http://git-wip-us.apache.org/repos/asf/ambari/blob/2516a059/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
index 9d3ce4c..5fe534a 100644
--- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
@@ -1203,7 +1203,7 @@ class TestHDP22StackAdvisor(TestCase):
'hive.server2.tez.default.queues': "queue1,queue2",
'hive.server2.tez.initialize.default.sessions': 'false',
'hive.server2.tez.sessions.per.default.queue': '1',
- 'hive.auto.convert.join.noconditionaltask.size': '268435456',
+ 'hive.auto.convert.join.noconditionaltask.size': '214748364',
'hive.compactor.initiator.on': 'false',
'hive.compactor.worker.threads': '0',
'hive.compute.query.using.stats': 'true',
@@ -1238,7 +1238,7 @@ class TestHDP22StackAdvisor(TestCase):
'hive.security.authorization.manager': 'org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdConfOnlyAuthorizerFactory'
},
'property_attributes': {
- 'hive.auto.convert.join.noconditionaltask.size': {'maximum': '805306368'},
+ 'hive.auto.convert.join.noconditionaltask.size': {'maximum': '644245094'},
'hive.tez.container.size': {'maximum': '8192', 'minimum': '256'},
'hive.server2.authentication.pam.services': {'delete': 'true'},
'hive.server2.custom.authentication.class': {'delete': 'true'},
http://git-wip-us.apache.org/repos/asf/ambari/blob/2516a059/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
index faad01c..34bebe8 100644
--- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
@@ -617,7 +617,7 @@ class TestHDP23StackAdvisor(TestCase):
'hive.server2.tez.default.queues': "queue1,queue2",
'hive.server2.tez.initialize.default.sessions': 'false',
'hive.server2.tez.sessions.per.default.queue': '1',
- 'hive.auto.convert.join.noconditionaltask.size': '268435456',
+ 'hive.auto.convert.join.noconditionaltask.size': '214748364',
'hive.compactor.initiator.on': 'false',
'hive.compactor.worker.threads': '0',
'hive.compute.query.using.stats': 'true',
@@ -652,7 +652,7 @@ class TestHDP23StackAdvisor(TestCase):
'hive.security.authorization.manager': 'org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdConfOnlyAuthorizerFactory'
},
'property_attributes': {
- 'hive.auto.convert.join.noconditionaltask.size': {'maximum': '805306368'},
+ 'hive.auto.convert.join.noconditionaltask.size': {'maximum': '644245094'},
'hive.server2.authentication.pam.services': {'delete': 'true'},
'hive.server2.custom.authentication.class': {'delete': 'true'},
'hive.server2.authentication.kerberos.principal': {'delete': 'true'},
@@ -884,7 +884,7 @@ class TestHDP23StackAdvisor(TestCase):
'hive.server2.tez.default.queues': "queue1,queue2",
'hive.server2.tez.initialize.default.sessions': 'false',
'hive.server2.tez.sessions.per.default.queue': '1',
- 'hive.auto.convert.join.noconditionaltask.size': '268435456',
+ 'hive.auto.convert.join.noconditionaltask.size': '214748364',
'hive.compactor.initiator.on': 'false',
'hive.compactor.worker.threads': '0',
'hive.compute.query.using.stats': 'true',
@@ -919,7 +919,7 @@ class TestHDP23StackAdvisor(TestCase):
'hive.security.authorization.manager': 'org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdConfOnlyAuthorizerFactory'
},
'property_attributes': {
- 'hive.auto.convert.join.noconditionaltask.size': {'maximum': '805306368'},
+ 'hive.auto.convert.join.noconditionaltask.size': {'maximum': '644245094'},
'hive.tez.container.size': {'maximum': '8192', 'minimum': '256'},
'hive.server2.authentication.pam.services': {'delete': 'true'},
'hive.server2.custom.authentication.class': {'delete': 'true'},
@@ -1161,7 +1161,7 @@ class TestHDP23StackAdvisor(TestCase):
"tez.task.resource.memory.mb": "768",
"tez.am.launch.cmd-opts": "-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseParallelGC",
"tez.task.launch.cmd-opts": "-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseParallelGC",
- "tez.runtime.io.sort.mb": "307",
+ "tez.runtime.io.sort.mb": "202",
"tez.session.am.dag.submit.timeout.secs": "600",
"tez.runtime.unordered.output.buffer.size-mb": "57",
"tez.am.resource.memory.mb": "4000",