You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by nc...@apache.org on 2017/02/13 22:13:25 UTC
[17/50] [abbrv] ambari git commit: AMBARI-19947. To make sure LLAP
preemption happens correctly,
should enable priority-utilization.underutilized-preemption.enabled=true when
YARN preemption is enabled.(vbrodetskyi)
AMBARI-19947. To make sure LLAP preemption happens correctly, should enable priority-utilization.underutilized-preemption.enabled=true when YARN preemption is enabled.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/59545f7f
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/59545f7f
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/59545f7f
Branch: refs/heads/branch-feature-AMBARI-12556
Commit: 59545f7fde971c352c7f7f44a19f436887fcdfe7
Parents: b15b606
Author: Vitaly Brodetskyi <vb...@hortonworks.com>
Authored: Fri Feb 10 14:41:12 2017 +0200
Committer: Vitaly Brodetskyi <vb...@hortonworks.com>
Committed: Fri Feb 10 14:41:12 2017 +0200
----------------------------------------------------------------------
.../YARN/3.0.0.3.0/service_advisor.py | 15 ++++++++++++++-
.../2.6/services/YARN/configuration/yarn-site.xml | 12 ++++++++++++
.../stacks/HDP/2.6/services/stack_advisor.py | 12 ++++++++++++
.../3.0/services/YARN/configuration/yarn-site.xml | 13 +++++++++++++
4 files changed, 51 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/59545f7f/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py
index 30dd474..aecf1e3 100644
--- a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py
+++ b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py
@@ -130,6 +130,7 @@ class YARNServiceAdvisor(service_advisor.ServiceAdvisor):
recommender.recommendYARNConfigurationsFromHDP22(configurations, clusterData, services, hosts)
recommender.recommendYARNConfigurationsFromHDP23(configurations, clusterData, services, hosts)
recommender.recommendYARNConfigurationsFromHDP25(configurations, clusterData, services, hosts)
+ recommender.recommendYARNConfigurationsFromHDP26(configurations, clusterData, services, hosts)
def getServiceConfigurationsValidationItems(self, configurations, recommendedDefaults, services, hosts):
"""
@@ -435,6 +436,18 @@ class YARNRecommender(service_advisor.ServiceAdvisor):
putYarnSiteProperty('yarn.timeline-service.entity-group-fs-store.group-id-plugin-classes', ",".join(timeline_plugin_classes_values))
putYarnSiteProperty('yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath', ":".join(timeline_plugin_classpath_values))
+
+ def recommendYARNConfigurationsFromHDP26(self, configurations, clusterData, services, hosts):
+ putYarnSiteProperty = self.putProperty(configurations, "yarn-site", services)
+
+ if "yarn-site" in services["configurations"] and \
+ "yarn.resourcemanager.scheduler.monitor.enable" in services["configurations"]["yarn-site"]["properties"]:
+ scheduler_monitor_enabled = services["configurations"]["yarn-site"]["properties"]["yarn.resourcemanager.scheduler.monitor.enable"]
+ if scheduler_monitor_enabled.lower() == 'true':
+ putYarnSiteProperty('yarn.scheduler.capacity.ordering-policy.priority-utilization.underutilized-preemption.enabled', "true")
+ else:
+ putYarnSiteProperty('yarn.scheduler.capacity.ordering-policy.priority-utilization.underutilized-preemption.enabled', "false")
+
#region LLAP
def updateLlapConfigs(self, configurations, services, hosts, llap_queue_name):
"""
@@ -1787,4 +1800,4 @@ class MAPREDUCE2Validator(service_advisor.ServiceAdvisor):
if yarnAppMapreduceAmCommandOpts > yarnAppMapreduceAmResourceMb:
validationItems.append({"config-name": 'yarn.app.mapreduce.am.command-opts', "item": self.getWarnItem("yarn.app.mapreduce.am.command-opts Xmx should be less than yarn.app.mapreduce.am.resource.mb ({0})".format(yarnAppMapreduceAmResourceMb))})
- return self.toConfigurationValidationProblems(validationItems, "mapred-site")
\ No newline at end of file
+ return self.toConfigurationValidationProblems(validationItems, "mapred-site")
http://git-wip-us.apache.org/repos/asf/ambari/blob/59545f7f/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-site.xml
index 58b528e..70a2cbe 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-site.xml
@@ -23,4 +23,16 @@
<description>When HA is enabled, the class to be used by Clients, AMs and NMs to failover to the Active RM. It should extend org.apache.hadoop.yarn.client.RMFailoverProxyProvider</description>
<on-ambari-upgrade add="false"/>
</property>
+ <property>
+ <name>yarn.scheduler.capacity.ordering-policy.priority-utilization.underutilized-preemption.enabled</name>
+ <value>false</value>
+ <description>This property need to be set to make preemption can happen under a fragmented cluster.</description>
+ <depends-on>
+ <property>
+ <type>yarn-site</type>
+ <name>yarn.resourcemanager.scheduler.monitor.enable</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="false"/>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/59545f7f/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
index e6ebd6f..969c3dd 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
@@ -138,6 +138,18 @@ class HDP26StackAdvisor(HDP25StackAdvisor):
elif superset_database_type == "postgresql":
putSupersetProperty("SUPERSET_DATABASE_PORT", "5432")
+ def recommendYARNConfigurations(self, configurations, clusterData, services, hosts):
+ super(HDP26StackAdvisor, self).recommendYARNConfigurations(configurations, clusterData, services, hosts)
+ putYarnSiteProperty = self.putProperty(configurations, "yarn-site", services)
+
+ if "yarn-site" in services["configurations"] and \
+ "yarn.resourcemanager.scheduler.monitor.enable" in services["configurations"]["yarn-site"]["properties"]:
+ scheduler_monitor_enabled = services["configurations"]["yarn-site"]["properties"]["yarn.resourcemanager.scheduler.monitor.enable"]
+ if scheduler_monitor_enabled.lower() == 'true':
+ putYarnSiteProperty('yarn.scheduler.capacity.ordering-policy.priority-utilization.underutilized-preemption.enabled', "true")
+ else:
+ putYarnSiteProperty('yarn.scheduler.capacity.ordering-policy.priority-utilization.underutilized-preemption.enabled', "false")
+
def getMetadataConnectionString(self, database_type):
driverDict = {
'mysql': 'jdbc:mysql://{0}:{2}/{1}?createDatabaseIfNotExist=true',
http://git-wip-us.apache.org/repos/asf/ambari/blob/59545f7f/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/yarn-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/yarn-site.xml b/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/yarn-site.xml
index 0f46d75..01551c4 100644
--- a/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/yarn-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/yarn-site.xml
@@ -25,6 +25,19 @@
<on-ambari-upgrade add="false"/>
</property>
+ <property>
+ <name>yarn.scheduler.capacity.ordering-policy.priority-utilization.underutilized-preemption.enabled</name>
+ <value>false</value>
+ <description>This property need to be set to make preemption can happen under a fragmented cluster.</description>
+ <depends-on>
+ <property>
+ <type>yarn-site</type>
+ <name>yarn.resourcemanager.scheduler.monitor.enable</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
<!-- These configs were inherited from HDP 2.5 -->
<property>
<name>yarn.nodemanager.aux-services.spark2_shuffle.classpath</name>