You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jo...@apache.org on 2017/04/24 02:21:18 UTC
[36/45] ambari git commit: AMBARI-20821. Update default values for
YARN preemption in Ambari.(vbrodetskyi)
AMBARI-20821. Update default values for YARN preemption in Ambari.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/99f16607
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/99f16607
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/99f16607
Branch: refs/heads/branch-feature-AMBARI-12556
Commit: 99f16607f804992b4c06ea3f0e9295399604e7f6
Parents: 84a8b76
Author: Vitaly Brodetskyi <vb...@hortonworks.com>
Authored: Fri Apr 21 18:53:06 2017 +0300
Committer: Vitaly Brodetskyi <vb...@hortonworks.com>
Committed: Fri Apr 21 18:53:58 2017 +0300
----------------------------------------------------------------------
.../YARN/3.0.0.3.0/configuration/yarn-site.xml | 12 ++++++++++++
.../common-services/YARN/3.0.0.3.0/service_advisor.py | 4 ++++
.../HDP/2.6/services/YARN/configuration/yarn-site.xml | 12 ++++++++++++
.../resources/stacks/HDP/2.6/services/stack_advisor.py | 5 +++++
.../test/python/stacks/2.6/common/test_stack_advisor.py | 3 +++
5 files changed, 36 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/99f16607/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/configuration/yarn-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/configuration/yarn-site.xml b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/configuration/yarn-site.xml
index 394fae3..cc96cd7 100644
--- a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/configuration/yarn-site.xml
+++ b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/configuration/yarn-site.xml
@@ -1177,4 +1177,16 @@ yarn.node-labels.manager-class
</value-attributes>
<on-ambari-upgrade add="false"/>
</property>
+ <property>
+ <name>yarn.resourcemanager.monitor.capacity.preemption.total_preemption_per_round</name>
+ <value>0.1</value>
+ <description>This option controls the pace at which containers-marked-for-preemption are actually preempted in each period.</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.monitor.capacity.preemption.natural_termination_factor</name>
+ <value>1</value>
+ <description>Similar to total_preemption_per_round, we can apply this factor to slowdown resource preemption after preemption-target is computed for each queue.</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/99f16607/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py
index 1ac7849..34fe117 100644
--- a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py
+++ b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py
@@ -430,6 +430,10 @@ class YARNRecommender(service_advisor.ServiceAdvisor):
else:
putYarnSiteProperty('yarn.scheduler.capacity.ordering-policy.priority-utilization.underutilized-preemption.enabled', "false")
+ # calculate total_preemption_per_round
+ total_preemption_per_round = str(round(max(float(1)/len(hosts['items']), 0.1),2))
+ putYarnSiteProperty('yarn.resourcemanager.monitor.capacity.preemption.total_preemption_per_round', total_preemption_per_round)
+
if 'yarn-env' in services['configurations'] and 'yarn_user' in services['configurations']['yarn-env']['properties']:
yarn_user = services['configurations']['yarn-env']['properties']['yarn_user']
else:
http://git-wip-us.apache.org/repos/asf/ambari/blob/99f16607/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-site.xml
index acdae65..c434c12 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-site.xml
@@ -42,6 +42,18 @@
<on-ambari-upgrade add="false"/>
</property>
<property>
+ <name>yarn.resourcemanager.monitor.capacity.preemption.total_preemption_per_round</name>
+ <value>0.1</value>
+ <description>This option controls the pace at which containers-marked-for-preemption are actually preempted in each period.</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.monitor.capacity.preemption.natural_termination_factor</name>
+ <value>1</value>
+ <description>Similar to total_preemption_per_round, we can apply this factor to slowdown resource preemption after preemption-target is computed for each queue.</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
<name>yarn.timeline-service.entity-group-fs-store.app-cache-size</name>
<value>10</value>
<description>
http://git-wip-us.apache.org/repos/asf/ambari/blob/99f16607/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
index 4e1b4b6..da3c98f 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
@@ -189,6 +189,11 @@ class HDP26StackAdvisor(HDP25StackAdvisor):
else:
putYarnSiteProperty('yarn.scheduler.capacity.ordering-policy.priority-utilization.underutilized-preemption.enabled', "false")
+ # calculate total_preemption_per_round
+ total_preemption_per_round = str(round(max(float(1)/len(hosts['items']), 0.1),2))
+ putYarnSiteProperty('yarn.resourcemanager.monitor.capacity.preemption.total_preemption_per_round', total_preemption_per_round)
+
+
if 'yarn-env' in services['configurations'] and 'yarn_user' in services['configurations']['yarn-env']['properties']:
yarn_user = services['configurations']['yarn-env']['properties']['yarn_user']
else:
http://git-wip-us.apache.org/repos/asf/ambari/blob/99f16607/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py
index 114dd3a..d4d28c9 100644
--- a/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py
@@ -1162,6 +1162,7 @@ class TestHDP26StackAdvisor(TestCase):
'yarn.timeline-service.entity-group-fs-store.app-cache-size': '10',
'yarn.scheduler.minimum-allocation-mb': '256',
'yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath': '',
+ 'yarn.resourcemanager.monitor.capacity.preemption.total_preemption_per_round': '1.0',
'yarn.nodemanager.resource.cpu-vcores': '4',
'yarn.scheduler.maximum-allocation-mb': '768',
'yarn.nodemanager.linux-container-executor.group': 'hadoop',
@@ -1336,6 +1337,7 @@ class TestHDP26StackAdvisor(TestCase):
'yarn.nodemanager.resource.memory-mb': '1280',
'yarn.scheduler.minimum-allocation-mb': '256',
'yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath': '',
+ 'yarn.resourcemanager.monitor.capacity.preemption.total_preemption_per_round': '1.0',
'yarn.nodemanager.resource.cpu-vcores': '4',
'yarn.scheduler.maximum-allocation-mb': '1280',
'yarn.nodemanager.linux-container-executor.group': 'hadoop',
@@ -1442,6 +1444,7 @@ class TestHDP26StackAdvisor(TestCase):
'yarn.nodemanager.resource.memory-mb': '1280',
'yarn.scheduler.minimum-allocation-mb': '256',
'yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath': '',
+ 'yarn.resourcemanager.monitor.capacity.preemption.total_preemption_per_round': '1.0',
'yarn.nodemanager.resource.cpu-vcores': '4',
'yarn.scheduler.maximum-allocation-mb': '1280',
'yarn.nodemanager.linux-container-executor.group': 'hadoop',