You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sm...@apache.org on 2017/04/05 19:56:12 UTC
[1/2] ambari git commit: AMBARI-20537. Some config changes for LLAP
to avoid daemon getting killed (Prasanth Jayachandran via smohanty)
Repository: ambari
Updated Branches:
refs/heads/branch-2.5 4bb66330c -> 44b5155fc
AMBARI-20537. Some config changes for LLAP to avoid daemon getting killed (Prasanth Jayachandran via smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/84ba22a7
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/84ba22a7
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/84ba22a7
Branch: refs/heads/branch-2.5
Commit: 84ba22a711bf89b3286c155ec99f638414079d6b
Parents: 4bb6633
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Thu Mar 23 18:11:55 2017 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Wed Apr 5 12:49:41 2017 -0700
----------------------------------------------------------------------
.../2.5/services/HIVE/configuration/hive-interactive-env.xml | 2 +-
.../2.5/services/HIVE/configuration/tez-interactive-site.xml | 5 +++++
.../2.6/services/HIVE/configuration/hive-interactive-env.xml | 2 +-
3 files changed, 7 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/84ba22a7/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml
index 07d21af..f3e7be1 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml
@@ -201,7 +201,7 @@
</property>
<property>
<name>llap_java_opts</name>
- <value>-XX:+AlwaysPreTouch {% if java_version > 7 %}-XX:+UseG1GC -XX:TLABSize=8m -XX:+ResizeTLAB -XX:+UseNUMA -XX:+AggressiveOpts -XX:MetaspaceSize=1024m -XX:InitiatingHeapOccupancyPercent=40 -XX:G1ReservePercent=20 -XX:MaxGCPauseMillis=200{% else %}-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseParallelGC{% endif %}</value>
+ <value>-XX:+AlwaysPreTouch {% if java_version > 7 %}-Xss512k -XX:+UseG1GC -XX:TLABSize=8m -XX:+ResizeTLAB -XX:+UseNUMA -XX:+AggressiveOpts -XX:MetaspaceSize=1024m -XX:InitiatingHeapOccupancyPercent=40 -XX:G1ReservePercent=20 -XX:MaxGCPauseMillis=200{% else %}-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseParallelGC{% endif %}</value>
<description>Java opts for llap daemons</description>
<display-name>LLAP daemon java opts</display-name>
<on-ambari-upgrade add="true"/>
http://git-wip-us.apache.org/repos/asf/ambari/blob/84ba22a7/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml
index e914d69..c1a42b0 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml
@@ -178,4 +178,9 @@
<value>3600</value>
<on-ambari-upgrade add="true"/>
</property>
+ <property>
+ <name>tez.runtime.shuffle.parallel.copies</name>
+ <value>8</value>
+ <on-ambari-upgrade add="false"/>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/84ba22a7/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-env.xml
index a99caba..ada4859 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-env.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-env.xml
@@ -44,7 +44,7 @@
</property>
<property>
<name>llap_java_opts</name>
- <value>-XX:+AlwaysPreTouch {% if java_version > 7 %}-XX:+UseG1GC -XX:TLABSize=8m -XX:+ResizeTLAB -XX:+UseNUMA -XX:+AggressiveOpts -XX:InitiatingHeapOccupancyPercent=40 -XX:G1ReservePercent=20 -XX:MaxGCPauseMillis=200{% else %}-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseParallelGC{% endif %}{{heap_dump_opts}}</value>
+ <value>-XX:+AlwaysPreTouch {% if java_version > 7 %}-Xss512k -XX:+UseG1GC -XX:TLABSize=8m -XX:+ResizeTLAB -XX:+UseNUMA -XX:+AggressiveOpts -XX:InitiatingHeapOccupancyPercent=40 -XX:G1ReservePercent=20 -XX:MaxGCPauseMillis=200{% else %}-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseParallelGC{% endif %}{{heap_dump_opts}}</value>
<description>Java opts for llap application</description>
<display-name>LLAP app java opts</display-name>
<on-ambari-upgrade add="false"/>
[2/2] ambari git commit: AMBARI-20554. Fix minimum recommended value
for slider and Tez AM for HSI.
Posted by sm...@apache.org.
AMBARI-20554. Fix minimum recommended value for slider and Tez AM for HSI.
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/44b5155f
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/44b5155f
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/44b5155f
Branch: refs/heads/branch-2.5
Commit: 44b5155fc0bdebd359143063524d90f14bf8572a
Parents: 84ba22a
Author: Swapan Shridhar <ss...@hortonworks.com>
Authored: Thu Mar 23 18:29:06 2017 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Wed Apr 5 12:50:34 2017 -0700
----------------------------------------------------------------------
.../stacks/HDP/2.5/services/stack_advisor.py | 10 ++++------
.../stacks/2.5/common/test_stack_advisor.py | 18 +++++++++---------
2 files changed, 13 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/44b5155f/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
index 3100125..b6aca4c 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
@@ -1495,12 +1495,10 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
:type yarn_min_container_size int
"""
- if yarn_min_container_size > 1024:
+ if yarn_min_container_size >= 1024:
return 1024
- if yarn_min_container_size >= 256 and yarn_min_container_size <= 1024:
- return yarn_min_container_size
- if yarn_min_container_size < 256:
- return 256
+ else:
+ return 512
def calculate_tez_am_container_size(self, services, total_cluster_capacity, is_cluster_create_opr=False, enable_hive_interactive_1st_invocation=False):
"""
@@ -1511,7 +1509,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
calculated_tez_am_resource_memory_mb = None
if is_cluster_create_opr or enable_hive_interactive_1st_invocation:
if total_cluster_capacity <= 4096:
- calculated_tez_am_resource_memory_mb = 256
+ calculated_tez_am_resource_memory_mb = 512
elif total_cluster_capacity > 4096 and total_cluster_capacity <= 98304:
calculated_tez_am_resource_memory_mb = 1024
elif total_cluster_capacity > 98304:
http://git-wip-us.apache.org/repos/asf/ambari/blob/44b5155f/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
index 42047d9..8576dcc 100644
--- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
@@ -1401,19 +1401,19 @@ class TestHDP25StackAdvisor(TestCase):
self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes'], 3)
self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes_for_llap_daemons'], 3)
- self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'], '9548')
+ self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'], '9207')
self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors'], '1')
self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.threadpool.size'], '1')
- self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size'], '8524')
+ self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size'], '8183')
self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.enabled'], 'true')
self.assertEqual(configurations['hive-interactive-env']['properties']['llap_heap_size'], '819')
self.assertEqual(configurations['hive-interactive-env']['properties']['hive_heapsize'], '2048')
self.assertEqual(configurations['hive-interactive-env']['property_attributes']['num_llap_nodes'], {'read_only': 'true', 'minimum': '1', 'maximum': '1'})
- self.assertEqual(configurations['hive-interactive-env']['properties']['slider_am_container_mb'], '341')
+ self.assertEqual(configurations['hive-interactive-env']['properties']['slider_am_container_mb'], '682')
self.assertEqual(configurations['hive-interactive-site']['properties']['hive.auto.convert.join.noconditionaltask.size'], '286261248')
self.assertTrue('tez.am.resource.memory.mb' not in configurations['tez-interactive-site']['properties'])
@@ -2007,19 +2007,19 @@ class TestHDP25StackAdvisor(TestCase):
self.assertTrue('num_llap_nodes' not in configurations['hive-interactive-env']['properties'])
self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes_for_llap_daemons'], 3)
- self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'], '10571')
+ self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'], '10230')
self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors'], '3')
self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.threadpool.size'], '3')
- self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size'], '1355')
+ self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size'], '1014')
self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.enabled'], 'true')
self.assertEqual(configurations['hive-interactive-env']['properties']['llap_heap_size'], '7372')
self.assertEqual(configurations['hive-interactive-env']['properties']['hive_heapsize'], '2048')
self.assertEqual(configurations['hive-interactive-env']['property_attributes']['num_llap_nodes'], {'maximum': '3', 'minimum': '1', 'read_only': 'false'})
- self.assertEqual(configurations['hive-interactive-env']['properties']['slider_am_container_mb'], '341')
+ self.assertEqual(configurations['hive-interactive-env']['properties']['slider_am_container_mb'], '682')
self.assertEqual(configurations['hive-interactive-site']['properties']['hive.server2.tez.default.queues'], 'llap')
self.assertEqual(configurations['hive-interactive-site']['properties']['hive.auto.convert.join.noconditionaltask.size'], '858783744')
@@ -2815,19 +2815,19 @@ class TestHDP25StackAdvisor(TestCase):
self.assertTrue('num_llap_nodes_for_llap_daemons' not in configurations['hive-interactive-env']['properties'])
self.assertTrue('num_llap_nodes' not in configurations['hive-interactive-env']['properties'])
- self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'], '199826')
+ self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'], '199485')
self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors'], '10')
self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.threadpool.size'], '10')
- self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size'], '158866')
+ self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size'], '158525')
self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.enabled'], 'true')
self.assertEqual(configurations['hive-interactive-env']['properties']['llap_heap_size'], '34816')
self.assertEqual(configurations['hive-interactive-env']['properties']['hive_heapsize'], '2048')
self.assertEqual(configurations['hive-interactive-env']['property_attributes']['num_llap_nodes'], {'maximum': '5', 'minimum': '1', 'read_only': 'false'})
- self.assertEqual(configurations['hive-interactive-env']['properties']['slider_am_container_mb'], '341')
+ self.assertEqual(configurations['hive-interactive-env']['properties']['slider_am_container_mb'], '682')
self.assertEqual(configurations['hive-interactive-site']['properties']['hive.server2.tez.default.queues'], 'llap')
self.assertEqual(configurations['hive-interactive-site']['properties']['hive.auto.convert.join.noconditionaltask.size'], '1145044992')